From d507bc3b0541581fc9960f157968e80c02247255 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 8 Jun 2022 19:38:46 +0100 Subject: [PATCH 01/55] target/arm: Declare support for FEAT_RASv1p1 The architectural feature RASv1p1 introduces the following new features: * new registers ERXPFGCDN_EL1, ERXPFGCTL_EL1 and ERXPFGF_EL1 * new bits in the fine-grained trap registers that control traps for these new registers * new trap bits HCR_EL2.FIEN and SCR_EL3.FIEN that control traps for ERXPFGCDN_EL1, ERXPFGCTL_EL1, ERXPFGP_EL1 * a larger number of the ERXMISC_EL1 registers * the format of ERRSTATUS registers changes The architecture permits that if ERRIDR_EL1.NUM is 0 (as it is for QEMU) then all these new registers may UNDEF, and the HCR_EL2.FIEN and SCR_EL3.FIEN bits may be RES0. We don't have any ERRSTATUS registers (again, because ERRIDR_EL1.NUM is 0). QEMU does not yet implement the fine-grained-trap extension. So there is nothing we need to implement to be compliant with the feature spec. Make the 'max' CPU report the feature in its ID registers, and document it. Signed-off-by: Peter Maydell Reviewed-by: Richard Henderson Message-id: 20220531114258.855804-1-peter.maydell@linaro.org --- docs/system/arm/emulation.rst | 1 + target/arm/cpu64.c | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index 49cc3e8340..81467f02ce 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -52,6 +52,7 @@ the following architecture extensions: - FEAT_PMUv3p1 (PMU Extensions v3.1) - FEAT_PMUv3p4 (PMU Extensions v3.4) - FEAT_RAS (Reliability, availability, and serviceability) +- FEAT_RASv1p1 (RAS Extension v1.1) - FEAT_RDM (Advanced SIMD rounding double multiply accumulate instructions) - FEAT_RNG (Random number generator) - FEAT_S2FWB (Stage 2 forced Write-Back) diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index 3ff9219ca3..bd1c62a342 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -916,6 +916,7 @@ static void aarch64_max_initfn(Object *obj) * we do for EL2 with the virtualization=on property. */ t = FIELD_DP64(t, ID_AA64PFR1, MTE, 3); /* FEAT_MTE3 */ + t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 1); /* FEAT_RASv1p1 */ t = FIELD_DP64(t, ID_AA64PFR1, CSV2_FRAC, 0); /* FEAT_CSV2_2 */ cpu->isar.id_aa64pfr1 = t; From 7ac610206a0049460c392c9559b51246af0afd6d Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 8 Jun 2022 19:38:46 +0100 Subject: [PATCH 02/55] target/arm: Implement FEAT_DoubleFault The FEAT_DoubleFault extension adds the following: * All external aborts on instruction fetches and translation table walks for instruction fetches must be synchronous. For QEMU this is already true. * SCR_EL3 has a new bit NMEA which disables the masking of SError interrupts by PSTATE.A when the SError interrupt is taken to EL3. For QEMU we only need to make the bit writable, because we have no sources of SError interrupts. * SCR_EL3 has a new bit EASE which causes synchronous external aborts taken to EL3 to be taken at the same entry point as SError. (Note that this does not mean that they are SErrors for purposes of PSTATE.A masking or that the syndrome register reports them as SErrors: it just means that the vector offset is different.) * The existing SCTLR_EL3.IESB has an effective value of 1 when SCR_EL3.NMEA is 1. For QEMU this is a no-op because we don't need different behaviour based on IESB (we don't need to do anything to ensure that error exceptions are synchronized). So for QEMU the things we need to change are: * Make SCR_EL3.{NMEA,EASE} writable * When taking a synchronous external abort at EL3, adjust the vector entry point if SCR_EL3.EASE is set * Advertise the feature in the ID registers Signed-off-by: Peter Maydell Reviewed-by: Richard Henderson Message-id: 20220531151431.949322-1-peter.maydell@linaro.org --- docs/system/arm/emulation.rst | 1 + target/arm/cpu.h | 5 +++++ target/arm/cpu64.c | 4 ++-- target/arm/helper.c | 36 +++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index 81467f02ce..83b4410065 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -23,6 +23,7 @@ the following architecture extensions: - FEAT_Debugv8p2 (Debug changes for v8.2) - FEAT_Debugv8p4 (Debug changes for v8.4) - FEAT_DotProd (Advanced SIMD dot product instructions) +- FEAT_DoubleFault (Double Fault Extension) - FEAT_FCMA (Floating-point complex number instructions) - FEAT_FHM (Floating-point half-precision multiplication instructions) - FEAT_FP16 (Half-precision floating-point data processing) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index c1865ad5da..0ee1705a4f 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3952,6 +3952,11 @@ static inline bool isar_feature_aa64_ras(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, RAS) != 0; } +static inline bool isar_feature_aa64_doublefault(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, RAS) >= 2; +} + static inline bool isar_feature_aa64_sve(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, SVE) != 0; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index bd1c62a342..cce68dd82a 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -899,7 +899,7 @@ static void aarch64_max_initfn(Object *obj) t = cpu->isar.id_aa64pfr0; t = FIELD_DP64(t, ID_AA64PFR0, FP, 1); /* FEAT_FP16 */ t = FIELD_DP64(t, ID_AA64PFR0, ADVSIMD, 1); /* FEAT_FP16 */ - t = FIELD_DP64(t, ID_AA64PFR0, RAS, 1); /* FEAT_RAS */ + t = FIELD_DP64(t, ID_AA64PFR0, RAS, 2); /* FEAT_RASv1p1 + FEAT_DoubleFault */ t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1); t = FIELD_DP64(t, ID_AA64PFR0, SEL2, 1); /* FEAT_SEL2 */ t = FIELD_DP64(t, ID_AA64PFR0, DIT, 1); /* FEAT_DIT */ @@ -916,7 +916,7 @@ static void aarch64_max_initfn(Object *obj) * we do for EL2 with the virtualization=on property. */ t = FIELD_DP64(t, ID_AA64PFR1, MTE, 3); /* FEAT_MTE3 */ - t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 1); /* FEAT_RASv1p1 */ + t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 0); /* FEAT_RASv1p1 + FEAT_DoubleFault */ t = FIELD_DP64(t, ID_AA64PFR1, CSV2_FRAC, 0); /* FEAT_CSV2_2 */ cpu->isar.id_aa64pfr1 = t; diff --git a/target/arm/helper.c b/target/arm/helper.c index 40da63913c..7f2c14bea9 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -1776,6 +1776,9 @@ static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) if (cpu_isar_feature(aa64_scxtnum, cpu)) { valid_mask |= SCR_ENSCXT; } + if (cpu_isar_feature(aa64_doublefault, cpu)) { + valid_mask |= SCR_EASE | SCR_NMEA; + } } else { valid_mask &= ~(SCR_RW | SCR_ST); if (cpu_isar_feature(aa32_ras, cpu)) { @@ -10113,6 +10116,31 @@ static uint32_t cpsr_read_for_spsr_elx(CPUARMState *env) return ret; } +static bool syndrome_is_sync_extabt(uint32_t syndrome) +{ + /* Return true if this syndrome value is a synchronous external abort */ + switch (syn_get_ec(syndrome)) { + case EC_INSNABORT: + case EC_INSNABORT_SAME_EL: + case EC_DATAABORT: + case EC_DATAABORT_SAME_EL: + /* Look at fault status code for all the synchronous ext abort cases */ + switch (syndrome & 0x3f) { + case 0x10: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return true; + default: + return false; + } + default: + return false; + } +} + /* Handle exception entry to a target EL which is using AArch64 */ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) { @@ -10168,6 +10196,14 @@ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) switch (cs->exception_index) { case EXCP_PREFETCH_ABORT: case EXCP_DATA_ABORT: + /* + * FEAT_DoubleFault allows synchronous external aborts taken to EL3 + * to be taken to the SError vector entrypoint. + */ + if (new_el == 3 && (env->cp15.scr_el3 & SCR_EASE) && + syndrome_is_sync_extabt(env->exception.syndrome)) { + addr += 0x180; + } env->cp15.far_el[new_el] = env->exception.vaddress; qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n", env->cp15.far_el[new_el]); From 9323e79f10e5f5d8fffc3b307776173ca11faeae Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 03/55] Fix 'writeable' typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have about 30 instances of the typo/variant spelling 'writeable', and over 500 of the more common 'writable'. Standardize on the latter. Change produced with: sed -i -e 's/\([Ww][Rr][Ii][Tt]\)[Ee]\([Aa][Bb][Ll][Ee]\)/\1\2/g' $(git grep -il writeable) and then hand-undoing the instance in linux-headers/linux/kvm.h. Most of these changes are in comments or documentation; the exceptions are: * a local variable in accel/hvf/hvf-accel-ops.c * a local variable in accel/kvm/kvm-all.c * the PMCR_WRITABLE_MASK macro in target/arm/internals.h * the EPT_VIOLATION_GPA_WRITABLE macro in target/i386/hvf/vmcs.h (which is never used anywhere) * the AR_TYPE_WRITABLE_MASK macro in target/i386/hvf/vmx.h (which is never used anywhere) Signed-off-by: Peter Maydell Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Stefan Weil Message-id: 20220505095015.2714666-1-peter.maydell@linaro.org --- accel/hvf/hvf-accel-ops.c | 4 ++-- accel/kvm/kvm-all.c | 4 ++-- accel/tcg/user-exec.c | 6 +++--- docs/interop/vhost-user.rst | 2 +- docs/specs/vmgenid.txt | 4 ++-- hw/acpi/ghes.c | 2 +- hw/intc/arm_gicv3_cpuif.c | 2 +- hw/intc/arm_gicv3_dist.c | 2 +- hw/intc/arm_gicv3_redist.c | 4 ++-- hw/intc/riscv_aclint.c | 2 +- hw/intc/riscv_aplic.c | 2 +- hw/pci/shpc.c | 2 +- hw/scsi/mfi.h | 2 +- hw/sparc64/sun4u_iommu.c | 2 +- hw/timer/sse-timer.c | 2 +- python/qemu/machine/machine.py | 2 +- target/arm/gdbstub.c | 2 +- target/arm/helper.c | 4 ++-- target/arm/hvf/hvf.c | 4 ++-- target/arm/internals.h | 4 ++-- target/i386/cpu-sysemu.c | 2 +- target/i386/hvf/vmcs.h | 2 +- target/i386/hvf/vmx.h | 2 +- target/s390x/ioinst.c | 2 +- tests/tcg/x86_64/system/boot.S | 2 +- 25 files changed, 34 insertions(+), 34 deletions(-) diff --git a/accel/hvf/hvf-accel-ops.c b/accel/hvf/hvf-accel-ops.c index a70e2eb375..24913ca9c4 100644 --- a/accel/hvf/hvf-accel-ops.c +++ b/accel/hvf/hvf-accel-ops.c @@ -120,12 +120,12 @@ static void hvf_set_phys_mem(MemoryRegionSection *section, bool add) { hvf_slot *mem; MemoryRegion *area = section->mr; - bool writeable = !area->readonly && !area->rom_device; + bool writable = !area->readonly && !area->rom_device; hv_memory_flags_t flags; uint64_t page_size = qemu_real_host_page_size(); if (!memory_region_is_ram(area)) { - if (writeable) { + if (writable) { return; } else if (!memory_region_is_romd(area)) { /* diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 32e177bd26..a4c4863f53 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1346,13 +1346,13 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, KVMSlot *mem; int err; MemoryRegion *mr = section->mr; - bool writeable = !mr->readonly && !mr->rom_device; + bool writable = !mr->readonly && !mr->rom_device; hwaddr start_addr, size, slot_size, mr_offset; ram_addr_t ram_start_offset; void *ram; if (!memory_region_is_ram(mr)) { - if (writeable || !kvm_readonly_mem_allowed) { + if (writable || !kvm_readonly_mem_allowed) { return; } else if (!mr->romd_mode) { /* If the memory device is not in romd_mode, then we actually want diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index ac57324d4f..20ada5472b 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -101,10 +101,10 @@ MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write) * Return true if the write fault has been handled, and should be re-tried. * * Note that it is important that we don't call page_unprotect() unless - * this is really a "write to nonwriteable page" fault, because + * this is really a "write to nonwritable page" fault, because * page_unprotect() assumes that if it is called for an access to - * a page that's writeable this means we had two threads racing and - * another thread got there first and already made the page writeable; + * a page that's writable this means we had two threads racing and + * another thread got there first and already made the page writable; * so we will retry the access. If we were to call page_unprotect() * for some other kind of fault that should really be passed to the * guest, we'd end up in an infinite loop of retrying the faulting access. diff --git a/docs/interop/vhost-user.rst b/docs/interop/vhost-user.rst index a99ba4433c..d7cf904f7f 100644 --- a/docs/interop/vhost-user.rst +++ b/docs/interop/vhost-user.rst @@ -222,7 +222,7 @@ Virtio device config space :size: a 32-bit configuration space access size in bytes :flags: a 32-bit value: - - 0: Vhost front-end messages used for writeable fields + - 0: Vhost front-end messages used for writable fields - 1: Vhost front-end messages used for live migration :payload: Size bytes array holding the contents of the virtio diff --git a/docs/specs/vmgenid.txt b/docs/specs/vmgenid.txt index aa9f518676..80ff69f31c 100644 --- a/docs/specs/vmgenid.txt +++ b/docs/specs/vmgenid.txt @@ -153,7 +153,7 @@ change the contents of the memory at runtime, specifically when starting a backed-up or snapshotted image. In order to do this, QEMU must know the address that has been allocated. -The mechanism chosen for this memory sharing is writeable fw_cfg blobs. +The mechanism chosen for this memory sharing is writable fw_cfg blobs. These are data object that are visible to both QEMU and guests, and are addressable as sequential files. @@ -164,7 +164,7 @@ Two fw_cfg blobs are used in this case: /etc/vmgenid_guid - contains the actual VM Generation ID GUID - read-only to the guest /etc/vmgenid_addr - contains the address of the downloaded vmgenid blob - - writeable by the guest + - writable by the guest QEMU sends the following commands to the guest at startup: diff --git a/hw/acpi/ghes.c b/hw/acpi/ghes.c index 45d9a809cc..e9511d9b8f 100644 --- a/hw/acpi/ghes.c +++ b/hw/acpi/ghes.c @@ -249,7 +249,7 @@ void build_ghes_error_table(GArray *hardware_errors, BIOSLinker *linker) for (i = 0; i < ACPI_GHES_ERROR_SOURCE_COUNT; i++) { /* * Initialize the value of read_ack_register to 1, so GHES can be - * writeable after (re)boot. + * writable after (re)boot. * ACPI 6.2: 18.3.2.8 Generic Hardware Error Source version 2 * (GHESv2 - Type 10) */ diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c index 8867e2e496..8ca630e5ad 100644 --- a/hw/intc/arm_gicv3_cpuif.c +++ b/hw/intc/arm_gicv3_cpuif.c @@ -2047,7 +2047,7 @@ static void icc_ctlr_el3_write(CPUARMState *env, const ARMCPRegInfo *ri, cs->icc_ctlr_el1[GICV3_S] |= ICC_CTLR_EL1_CBPR; } - /* The only bit stored in icc_ctlr_el3 which is writeable is EOIMODE_EL3: */ + /* The only bit stored in icc_ctlr_el3 which is writable is EOIMODE_EL3: */ mask = ICC_CTLR_EL3_EOIMODE_EL3; cs->icc_ctlr_el3 &= ~mask; diff --git a/hw/intc/arm_gicv3_dist.c b/hw/intc/arm_gicv3_dist.c index b9ed955e36..eea0368118 100644 --- a/hw/intc/arm_gicv3_dist.c +++ b/hw/intc/arm_gicv3_dist.c @@ -611,7 +611,7 @@ static bool gicd_writel(GICv3State *s, hwaddr offset, if (value & mask & GICD_CTLR_DS) { /* We just set DS, so the ARE_NS and EnG1S bits are now RES0. * Note that this is a one-way transition because if DS is set - * then it's not writeable, so it can only go back to 0 with a + * then it's not writable, so it can only go back to 0 with a * hardware reset. */ s->gicd_ctlr &= ~(GICD_CTLR_EN_GRP1S | GICD_CTLR_ARE_NS); diff --git a/hw/intc/arm_gicv3_redist.c b/hw/intc/arm_gicv3_redist.c index c3d4cdd66b..f1ecb2502b 100644 --- a/hw/intc/arm_gicv3_redist.c +++ b/hw/intc/arm_gicv3_redist.c @@ -257,7 +257,7 @@ static void gicr_write_vpendbaser(GICv3CPUState *cs, uint64_t newval) /* * The DIRTY bit is read-only and for us is always zero; - * other fields are writeable. + * other fields are writable. */ newval &= R_GICR_VPENDBASER_INNERCACHE_MASK | R_GICR_VPENDBASER_SHAREABILITY_MASK | @@ -491,7 +491,7 @@ static MemTxResult gicr_writel(GICv3CPUState *cs, hwaddr offset, /* RAZ/WI for our implementation */ return MEMTX_OK; case GICR_WAKER: - /* Only the ProcessorSleep bit is writeable. When the guest sets + /* Only the ProcessorSleep bit is writable. When the guest sets * it it requests that we transition the channel between the * redistributor and the cpu interface to quiescent, and that * we set the ChildrenAsleep bit once the inteface has reached the diff --git a/hw/intc/riscv_aclint.c b/hw/intc/riscv_aclint.c index e6bceceefd..e7942c4e5a 100644 --- a/hw/intc/riscv_aclint.c +++ b/hw/intc/riscv_aclint.c @@ -463,7 +463,7 @@ static void riscv_aclint_swi_realize(DeviceState *dev, Error **errp) /* Claim software interrupt bits */ for (i = 0; i < swi->num_harts; i++) { RISCVCPU *cpu = RISCV_CPU(qemu_get_cpu(swi->hartid_base + i)); - /* We don't claim mip.SSIP because it is writeable by software */ + /* We don't claim mip.SSIP because it is writable by software */ if (riscv_cpu_claim_interrupts(cpu, swi->sswi ? 0 : MIP_MSIP) < 0) { error_report("MSIP already claimed"); exit(1); diff --git a/hw/intc/riscv_aplic.c b/hw/intc/riscv_aplic.c index e7809fb6b2..cfd007e629 100644 --- a/hw/intc/riscv_aplic.c +++ b/hw/intc/riscv_aplic.c @@ -646,7 +646,7 @@ static void riscv_aplic_write(void *opaque, hwaddr addr, uint64_t value, } if (addr == APLIC_DOMAINCFG) { - /* Only IE bit writeable at the moment */ + /* Only IE bit writable at the moment */ value &= APLIC_DOMAINCFG_IE; aplic->domaincfg = value; } else if ((APLIC_SOURCECFG_BASE <= addr) && diff --git a/hw/pci/shpc.c b/hw/pci/shpc.c index 28e62174c4..f822f18b98 100644 --- a/hw/pci/shpc.c +++ b/hw/pci/shpc.c @@ -456,7 +456,7 @@ static int shpc_cap_add_config(PCIDevice *d, Error **errp) pci_set_byte(config + SHPC_CAP_CxP, 0); pci_set_long(config + SHPC_CAP_DWORD_DATA, 0); d->shpc->cap = config_offset; - /* Make dword select and data writeable. */ + /* Make dword select and data writable. */ pci_set_byte(d->wmask + config_offset + SHPC_CAP_DWORD_SELECT, 0xff); pci_set_long(d->wmask + config_offset + SHPC_CAP_DWORD_DATA, 0xffffffff); return 0; diff --git a/hw/scsi/mfi.h b/hw/scsi/mfi.h index e67a5c0b47..0b4ee53dfc 100644 --- a/hw/scsi/mfi.h +++ b/hw/scsi/mfi.h @@ -633,7 +633,7 @@ struct mfi_ctrl_props { * metadata and user data * 1=5%, 2=10%, 3=15% and so on */ - uint8_t viewSpace; /* snapshot writeable VIEWs + uint8_t viewSpace; /* snapshot writable VIEWs * capacity as a % of source LD * capacity. 0=READ only * 1=5%, 2=10%, 3=15% and so on diff --git a/hw/sparc64/sun4u_iommu.c b/hw/sparc64/sun4u_iommu.c index 9178277f82..1c1dca712e 100644 --- a/hw/sparc64/sun4u_iommu.c +++ b/hw/sparc64/sun4u_iommu.c @@ -165,7 +165,7 @@ static IOMMUTLBEntry sun4u_translate_iommu(IOMMUMemoryRegion *iommu, } if (tte & IOMMU_TTE_DATA_W) { - /* Writeable */ + /* Writable */ ret.perm = IOMMU_RW; } else { ret.perm = IOMMU_RO; diff --git a/hw/timer/sse-timer.c b/hw/timer/sse-timer.c index f959cb9d60..e92e83747d 100644 --- a/hw/timer/sse-timer.c +++ b/hw/timer/sse-timer.c @@ -324,7 +324,7 @@ static void sse_timer_write(void *opaque, hwaddr offset, uint64_t value, { uint32_t old_ctl = s->cntp_aival_ctl; - /* EN bit is writeable; CLR bit is write-0-to-clear, write-1-ignored */ + /* EN bit is writable; CLR bit is write-0-to-clear, write-1-ignored */ s->cntp_aival_ctl &= ~R_CNTP_AIVAL_CTL_EN_MASK; s->cntp_aival_ctl |= value & R_CNTP_AIVAL_CTL_EN_MASK; if (!(value & R_CNTP_AIVAL_CTL_CLR_MASK)) { diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index 07ac5a710b..37191f433b 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -495,7 +495,7 @@ class QEMUMachine: """ # If we keep the console socket open, we may deadlock waiting # for QEMU to exit, while QEMU is waiting for the socket to - # become writeable. + # become writable. if self._console_socket is not None: self._console_socket.close() self._console_socket = None diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c index f5b35cd55f..2f806512d0 100644 --- a/target/arm/gdbstub.c +++ b/target/arm/gdbstub.c @@ -118,7 +118,7 @@ int arm_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n) /* * Don't allow writing to XPSR.Exception as it can cause * a transition into or out of handler mode (it's not - * writeable via the MSR insn so this is a reasonable + * writable via the MSR insn so this is a reasonable * restriction). Other fields are safe to update. */ xpsr_write(env, tmp, ~XPSR_EXCP); diff --git a/target/arm/helper.c b/target/arm/helper.c index 7f2c14bea9..5727ead5e4 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -1411,8 +1411,8 @@ static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri, } } - env->cp15.c9_pmcr &= ~PMCR_WRITEABLE_MASK; - env->cp15.c9_pmcr |= (value & PMCR_WRITEABLE_MASK); + env->cp15.c9_pmcr &= ~PMCR_WRITABLE_MASK; + env->cp15.c9_pmcr |= (value & PMCR_WRITABLE_MASK); pmu_op_finish(env); } diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c index 1fdc5eef92..060aa0ccf4 100644 --- a/target/arm/hvf/hvf.c +++ b/target/arm/hvf/hvf.c @@ -978,8 +978,8 @@ static int hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val) } } - env->cp15.c9_pmcr &= ~PMCR_WRITEABLE_MASK; - env->cp15.c9_pmcr |= (val & PMCR_WRITEABLE_MASK); + env->cp15.c9_pmcr &= ~PMCR_WRITABLE_MASK; + env->cp15.c9_pmcr |= (val & PMCR_WRITABLE_MASK); pmu_op_finish(env); break; diff --git a/target/arm/internals.h b/target/arm/internals.h index b654bee468..1e4887b2dd 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1280,10 +1280,10 @@ enum MVEECIState { #define PMCRP 0x2 #define PMCRE 0x1 /* - * Mask of PMCR bits writeable by guest (not including WO bits like C, P, + * Mask of PMCR bits writable by guest (not including WO bits like C, P, * which can be written as 1 to trigger behaviour but which stay RAZ). */ -#define PMCR_WRITEABLE_MASK (PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE) +#define PMCR_WRITABLE_MASK (PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE) #define PMXEVTYPER_P 0x80000000 #define PMXEVTYPER_U 0x40000000 diff --git a/target/i386/cpu-sysemu.c b/target/i386/cpu-sysemu.c index e254d8ba10..a6f47b7d11 100644 --- a/target/i386/cpu-sysemu.c +++ b/target/i386/cpu-sysemu.c @@ -103,7 +103,7 @@ static void x86_cpu_to_dict(X86CPU *cpu, QDict *props) /* Convert CPU model data from X86CPU object to a property dictionary * that can recreate exactly the same CPU model, including every - * writeable QOM property. + * writable QOM property. */ static void x86_cpu_to_dict_full(X86CPU *cpu, QDict *props) { diff --git a/target/i386/hvf/vmcs.h b/target/i386/hvf/vmcs.h index 42de7ebc3a..b4692f63f6 100644 --- a/target/i386/hvf/vmcs.h +++ b/target/i386/hvf/vmcs.h @@ -330,7 +330,7 @@ #define EPT_VIOLATION_DATA_WRITE (1UL << 1) #define EPT_VIOLATION_INST_FETCH (1UL << 2) #define EPT_VIOLATION_GPA_READABLE (1UL << 3) -#define EPT_VIOLATION_GPA_WRITEABLE (1UL << 4) +#define EPT_VIOLATION_GPA_WRITABLE (1UL << 4) #define EPT_VIOLATION_GPA_EXECUTABLE (1UL << 5) #define EPT_VIOLATION_GLA_VALID (1UL << 7) #define EPT_VIOLATION_XLAT_VALID (1UL << 8) diff --git a/target/i386/hvf/vmx.h b/target/i386/hvf/vmx.h index 573ddc33c0..fcd9a95e5b 100644 --- a/target/i386/hvf/vmx.h +++ b/target/i386/hvf/vmx.h @@ -80,7 +80,7 @@ static inline uint64_t cap2ctrl(uint64_t cap, uint64_t ctrl) #define AR_TYPE_ACCESSES_MASK 1 #define AR_TYPE_READABLE_MASK (1 << 1) -#define AR_TYPE_WRITEABLE_MASK (1 << 2) +#define AR_TYPE_WRITABLE_MASK (1 << 2) #define AR_TYPE_CODE_MASK (1 << 3) #define AR_TYPE_MASK 0x0f #define AR_TYPE_BUSY_64_TSS 11 diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c index bdae5090bc..b12f18d346 100644 --- a/target/s390x/ioinst.c +++ b/target/s390x/ioinst.c @@ -284,7 +284,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, g_assert(!s390_is_pv()); /* * As operand exceptions have a lower priority than access exceptions, - * we check whether the memory area is writeable (injecting the + * we check whether the memory area is writable (injecting the * access execption if it is not) first. */ if (!s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib))) { diff --git a/tests/tcg/x86_64/system/boot.S b/tests/tcg/x86_64/system/boot.S index f8a2fcc839..ed0f638406 100644 --- a/tests/tcg/x86_64/system/boot.S +++ b/tests/tcg/x86_64/system/boot.S @@ -56,7 +56,7 @@ * * - `ebx`: contains the physical memory address where the loader has placed * the boot start info structure. - * - `cr0`: bit 0 (PE) must be set. All the other writeable bits are cleared. + * - `cr0`: bit 0 (PE) must be set. All the other writable bits are cleared. * - `cr4`: all bits are cleared. * - `cs `: must be a 32-bit read/execute code segment with a base of ‘0’ * and a limit of ‘0xFFFFFFFF’. The selector value is unspecified. From d2008b3355061ab0cca823739cd98045e769a809 Mon Sep 17 00:00:00 2001 From: Frederic Konrad Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 04/55] xlnx_dp: fix the wrong register size The core and the vblend registers size are wrong, they should respectively be 0x3B0 and 0x1E0 according to: https://www.xilinx.com/htmldocs/registers/ug1087/ug1087-zynq-ultrascale-registers.html. Let's fix that and use macros when creating the mmio region. Fixes: 58ac482a66d ("introduce xlnx-dp") Signed-off-by: Frederic Konrad Reviewed-by: Edgar E. Iglesias Acked-by: Alistair Francis Message-id: 20220601172353.3220232-2-fkonrad@xilinx.com Signed-off-by: Peter Maydell --- hw/display/xlnx_dp.c | 17 ++++++++++------- include/hw/display/xlnx_dp.h | 9 +++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c index 9bb781e312..0378570459 100644 --- a/hw/display/xlnx_dp.c +++ b/hw/display/xlnx_dp.c @@ -1219,19 +1219,22 @@ static void xlnx_dp_init(Object *obj) SysBusDevice *sbd = SYS_BUS_DEVICE(obj); XlnxDPState *s = XLNX_DP(obj); - memory_region_init(&s->container, obj, TYPE_XLNX_DP, 0xC050); + memory_region_init(&s->container, obj, TYPE_XLNX_DP, DP_CONTAINER_SIZE); memory_region_init_io(&s->core_iomem, obj, &dp_ops, s, TYPE_XLNX_DP - ".core", 0x3AF); - memory_region_add_subregion(&s->container, 0x0000, &s->core_iomem); + ".core", sizeof(s->core_registers)); + memory_region_add_subregion(&s->container, DP_CORE_REG_OFFSET, + &s->core_iomem); memory_region_init_io(&s->vblend_iomem, obj, &vblend_ops, s, TYPE_XLNX_DP - ".v_blend", 0x1DF); - memory_region_add_subregion(&s->container, 0xA000, &s->vblend_iomem); + ".v_blend", sizeof(s->vblend_registers)); + memory_region_add_subregion(&s->container, DP_VBLEND_REG_OFFSET, + &s->vblend_iomem); memory_region_init_io(&s->avbufm_iomem, obj, &avbufm_ops, s, TYPE_XLNX_DP - ".av_buffer_manager", 0x238); - memory_region_add_subregion(&s->container, 0xB000, &s->avbufm_iomem); + ".av_buffer_manager", sizeof(s->avbufm_registers)); + memory_region_add_subregion(&s->container, DP_AVBUF_REG_OFFSET, + &s->avbufm_iomem); memory_region_init_io(&s->audio_iomem, obj, &audio_ops, s, TYPE_XLNX_DP ".audio", sizeof(s->audio_registers)); diff --git a/include/hw/display/xlnx_dp.h b/include/hw/display/xlnx_dp.h index 8ab4733bb8..1ef5a89ee7 100644 --- a/include/hw/display/xlnx_dp.h +++ b/include/hw/display/xlnx_dp.h @@ -39,10 +39,15 @@ #define AUD_CHBUF_MAX_DEPTH (32 * KiB) #define MAX_QEMU_BUFFER_SIZE (4 * KiB) -#define DP_CORE_REG_ARRAY_SIZE (0x3AF >> 2) +#define DP_CORE_REG_OFFSET (0x0000) +#define DP_CORE_REG_ARRAY_SIZE (0x3B0 >> 2) +#define DP_AVBUF_REG_OFFSET (0xB000) #define DP_AVBUF_REG_ARRAY_SIZE (0x238 >> 2) -#define DP_VBLEND_REG_ARRAY_SIZE (0x1DF >> 2) +#define DP_VBLEND_REG_OFFSET (0xA000) +#define DP_VBLEND_REG_ARRAY_SIZE (0x1E0 >> 2) +#define DP_AUDIO_REG_OFFSET (0xC000) #define DP_AUDIO_REG_ARRAY_SIZE (0x50 >> 2) +#define DP_CONTAINER_SIZE (0xC050) struct PixmanPlane { pixman_format_code_t format; From 759ae1b47e7e0203eb6881304b4bbf0d935fd550 Mon Sep 17 00:00:00 2001 From: Sai Pavan Boddu Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 05/55] xlnx_dp: Introduce a vblank signal Add a periodic timer which raises vblank at a frequency of 30Hz. Note that this is a migration compatibility break for the xlnx-zcu102 board type. Signed-off-by: Sai Pavan Boddu Signed-off-by: Edgar E. Iglesias Signed-off-by: Frederic Konrad Acked-by: Alistair Francis Message-id: 20220601172353.3220232-3-fkonrad@xilinx.com Changes by fkonrad: - Switched to transaction-based ptimer API. - Added the DP_INT_VBLNK_START macro. Signed-off-by: Frederic Konrad [PMM: bump vmstate version, add commit message note about compat break] Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- hw/display/xlnx_dp.c | 30 ++++++++++++++++++++++++++---- include/hw/display/xlnx_dp.h | 3 +++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c index 0378570459..ed856b596d 100644 --- a/hw/display/xlnx_dp.c +++ b/hw/display/xlnx_dp.c @@ -114,6 +114,7 @@ #define DP_TX_N_AUD (0x032C >> 2) #define DP_TX_AUDIO_EXT_DATA(n) ((0x0330 + 4 * n) >> 2) #define DP_INT_STATUS (0x03A0 >> 2) +#define DP_INT_VBLNK_START (1 << 13) #define DP_INT_MASK (0x03A4 >> 2) #define DP_INT_EN (0x03A8 >> 2) #define DP_INT_DS (0x03AC >> 2) @@ -260,7 +261,7 @@ typedef enum DPVideoFmt DPVideoFmt; static const VMStateDescription vmstate_dp = { .name = TYPE_XLNX_DP, - .version_id = 1, + .version_id = 2, .fields = (VMStateField[]){ VMSTATE_UINT32_ARRAY(core_registers, XlnxDPState, DP_CORE_REG_ARRAY_SIZE), @@ -270,10 +271,15 @@ static const VMStateDescription vmstate_dp = { DP_VBLEND_REG_ARRAY_SIZE), VMSTATE_UINT32_ARRAY(audio_registers, XlnxDPState, DP_AUDIO_REG_ARRAY_SIZE), + VMSTATE_PTIMER(vblank, XlnxDPState), VMSTATE_END_OF_LIST() } }; +#define DP_VBLANK_PTIMER_POLICY (PTIMER_POLICY_WRAP_AFTER_ONE_PERIOD | \ + PTIMER_POLICY_CONTINUOUS_TRIGGER | \ + PTIMER_POLICY_NO_IMMEDIATE_TRIGGER) + static void xlnx_dp_update_irq(XlnxDPState *s); static uint64_t xlnx_dp_audio_read(void *opaque, hwaddr offset, unsigned size) @@ -773,6 +779,13 @@ static void xlnx_dp_write(void *opaque, hwaddr offset, uint64_t value, break; case DP_TRANSMITTER_ENABLE: s->core_registers[offset] = value & 0x01; + ptimer_transaction_begin(s->vblank); + if (value & 0x1) { + ptimer_run(s->vblank, 0); + } else { + ptimer_stop(s->vblank); + } + ptimer_transaction_commit(s->vblank); break; case DP_FORCE_SCRAMBLER_RESET: /* @@ -1177,9 +1190,6 @@ static void xlnx_dp_update_display(void *opaque) return; } - s->core_registers[DP_INT_STATUS] |= (1 << 13); - xlnx_dp_update_irq(s); - xlnx_dpdma_trigger_vsync_irq(s->dpdma); /* @@ -1275,6 +1285,14 @@ static void xlnx_dp_finalize(Object *obj) fifo8_destroy(&s->rx_fifo); } +static void vblank_hit(void *opaque) +{ + XlnxDPState *s = XLNX_DP(opaque); + + s->core_registers[DP_INT_STATUS] |= DP_INT_VBLNK_START; + xlnx_dp_update_irq(s); +} + static void xlnx_dp_realize(DeviceState *dev, Error **errp) { XlnxDPState *s = XLNX_DP(dev); @@ -1309,6 +1327,10 @@ static void xlnx_dp_realize(DeviceState *dev, Error **errp) &as); AUD_set_volume_out(s->amixer_output_stream, 0, 255, 255); xlnx_dp_audio_activate(s); + s->vblank = ptimer_init(vblank_hit, s, DP_VBLANK_PTIMER_POLICY); + ptimer_transaction_begin(s->vblank); + ptimer_set_freq(s->vblank, 30); + ptimer_transaction_commit(s->vblank); } static void xlnx_dp_reset(DeviceState *dev) diff --git a/include/hw/display/xlnx_dp.h b/include/hw/display/xlnx_dp.h index 1ef5a89ee7..e86a87f235 100644 --- a/include/hw/display/xlnx_dp.h +++ b/include/hw/display/xlnx_dp.h @@ -35,6 +35,7 @@ #include "hw/dma/xlnx_dpdma.h" #include "audio/audio.h" #include "qom/object.h" +#include "hw/ptimer.h" #define AUD_CHBUF_MAX_DEPTH (32 * KiB) #define MAX_QEMU_BUFFER_SIZE (4 * KiB) @@ -107,6 +108,8 @@ struct XlnxDPState { */ DPCDState *dpcd; I2CDDCState *edid; + + ptimer_state *vblank; }; #define TYPE_XLNX_DP "xlnx.v-dp" From 39f40d02f6af8e4718100b6a3854e963c962cfd7 Mon Sep 17 00:00:00 2001 From: Sai Pavan Boddu Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 06/55] xlnx_dp: Fix the interrupt disable logic Fix interrupt disable logic. Mask value 1 indicates that interrupts are disabled. Signed-off-by: Sai Pavan Boddu Reviewed-by: Edgar E. Iglesias Signed-off-by: Frederic Konrad Acked-by: Alistair Francis Message-id: 20220601172353.3220232-4-fkonrad@xilinx.com Signed-off-by: Peter Maydell --- hw/display/xlnx_dp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c index ed856b596d..a071c81883 100644 --- a/hw/display/xlnx_dp.c +++ b/hw/display/xlnx_dp.c @@ -889,7 +889,7 @@ static void xlnx_dp_write(void *opaque, hwaddr offset, uint64_t value, xlnx_dp_update_irq(s); break; case DP_INT_DS: - s->core_registers[DP_INT_MASK] |= ~value; + s->core_registers[DP_INT_MASK] |= value; xlnx_dp_update_irq(s); break; default: From b3f5cc3fda5775b34aa52e111d50c9d10911b352 Mon Sep 17 00:00:00 2001 From: Frederic Konrad Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 07/55] xlnx-zynqmp: fix the irq mapping for the display port and its dma When the display port has been initially implemented the device driver wasn't using interrupts. Now that the display port driver waits for vblank interrupt it has been noticed that the irq mapping is wrong. So use the value from the linux device tree and the ultrascale+ reference manual. Signed-off-by: Frederic Konrad Reviewed-by: Edgar E. Iglesias Acked-by: Alistair Francis Message-id: 20220601172353.3220232-5-fkonrad@xilinx.com [PMM: refold lines in commit message] Signed-off-by: Peter Maydell --- hw/arm/xlnx-zynqmp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/arm/xlnx-zynqmp.c b/hw/arm/xlnx-zynqmp.c index 375309e68e..383e177a00 100644 --- a/hw/arm/xlnx-zynqmp.c +++ b/hw/arm/xlnx-zynqmp.c @@ -60,10 +60,10 @@ #define SERDES_SIZE 0x20000 #define DP_ADDR 0xfd4a0000 -#define DP_IRQ 113 +#define DP_IRQ 0x77 #define DPDMA_ADDR 0xfd4c0000 -#define DPDMA_IRQ 116 +#define DPDMA_IRQ 0x7a #define APU_ADDR 0xfd5c0000 #define APU_IRQ 153 From d8cca960a9a5dbd216d2331cef3cc72f2c7338e0 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 08/55] target/arm: Move stage_1_mmu_idx decl to internals.h Move the decl from ptw.h to internals.h. Provide an inline version for user-only, just as we do for arm_stage1_mmu_idx. Move an endif down to make the definition in helper.c be system only. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-2-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 5 ++--- target/arm/internals.h | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 5727ead5e4..829b660db9 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10517,12 +10517,10 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, } } -#endif /* !CONFIG_USER_ONLY */ - /* Convert a possible stage1+2 MMU index into the appropriate * stage 1 MMU index */ -static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) +ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) { switch (mmu_idx) { case ARMMMUIdx_SE10_0: @@ -10541,6 +10539,7 @@ static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) return mmu_idx; } } +#endif /* !CONFIG_USER_ONLY */ /* Return true if the translation regime is using LPAE format page tables */ static inline bool regime_using_lpae_format(CPUARMState *env, diff --git a/target/arm/internals.h b/target/arm/internals.h index 1e4887b2dd..049edce946 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -979,11 +979,16 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env); * Return the ARMMMUIdx for the stage1 traversal for the current regime. */ #ifdef CONFIG_USER_ONLY +static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) +{ + return ARMMMUIdx_Stage1_E0; +} static inline ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) { return ARMMMUIdx_Stage1_E0; } #else +ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx); ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env); #endif From 8ae0886002aaf917601701df933617b2449e94b5 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 09/55] target/arm: Move get_phys_addr to ptw.c Begin moving all of the page table walking functions out of helper.c, starting with get_phys_addr(). Create a temporary header file, "ptw.h", in which to share declarations between the two C files while we are moving functions. Move a few declarations to "internals.h", which will remain used by multiple C files. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-3-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 344 +++++------------------------------------ target/arm/internals.h | 18 ++- target/arm/meson.build | 1 + target/arm/ptw.c | 267 ++++++++++++++++++++++++++++++++ target/arm/ptw.h | 51 ++++++ 5 files changed, 372 insertions(+), 309 deletions(-) create mode 100644 target/arm/ptw.c create mode 100644 target/arm/ptw.h diff --git a/target/arm/helper.c b/target/arm/helper.c index 829b660db9..3ffd122178 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -37,22 +37,11 @@ #include "semihosting/common-semi.h" #endif #include "cpregs.h" +#include "ptw.h" #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */ -#ifndef CONFIG_USER_ONLY - -static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) - __attribute__((nonnull)); -#endif - static void switch_mode(CPUARMState *env, int mode); -static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx); static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri) { @@ -10440,17 +10429,10 @@ uint64_t arm_sctlr(CPUARMState *env, int el) return env->cp15.sctlr_el[el]; } -/* Return the SCTLR value which controls this address translation regime */ -static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; -} - #ifndef CONFIG_USER_ONLY /* Return true if the specified stage of address translation is disabled */ -static inline bool regime_translation_disabled(CPUARMState *env, - ARMMMUIdx mmu_idx) +bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) { uint64_t hcr_el2; @@ -10542,8 +10524,7 @@ ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) #endif /* !CONFIG_USER_ONLY */ /* Return true if the translation regime is using LPAE format page tables */ -static inline bool regime_using_lpae_format(CPUARMState *env, - ARMMMUIdx mmu_idx) +bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) { int el = regime_el(env, mmu_idx); if (el == 2 || arm_el_is_aa64(env, el)) { @@ -10567,7 +10548,7 @@ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) } #ifndef CONFIG_USER_ONLY -static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) +bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { case ARMMMUIdx_SE10_0: @@ -10959,11 +10940,11 @@ static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) +bool get_phys_addr_v5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) { CPUState *cs = env_cpu(env); int level = 1; @@ -11081,10 +11062,10 @@ do_fault: return true; } -static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, ARMMMUFaultInfo *fi) +bool get_phys_addr_v6(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, ARMMMUFaultInfo *fi) { CPUState *cs = env_cpu(env); ARMCPU *cpu = env_archcpu(env); @@ -11360,7 +11341,7 @@ unsigned int arm_pamax(ARMCPU *cpu) return pamax_map[parange]; } -static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) +int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { return extract64(tcr, 37, 2); @@ -11372,7 +11353,7 @@ static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) } } -static int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx) +int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { return extract64(tcr, 51, 2); @@ -11602,12 +11583,12 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, * @fi: set to fault info if the translation fails * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes */ -static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, + hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); @@ -12055,11 +12036,11 @@ static inline bool m_is_system_region(CPUARMState *env, uint32_t address) return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; } -static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) +bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) { ARMCPU *cpu = env_archcpu(env); int n; @@ -12501,11 +12482,11 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, } -static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, target_ulong *page_size, - ARMMMUFaultInfo *fi) +bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, target_ulong *page_size, + ARMMMUFaultInfo *fi) { uint32_t secure = regime_is_secure(env, mmu_idx); V8M_SAttributes sattrs = {}; @@ -12575,10 +12556,10 @@ static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, return ret; } -static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - ARMMMUFaultInfo *fi) +bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + ARMMMUFaultInfo *fi) { int n; uint32_t mask; @@ -12795,8 +12776,8 @@ static uint8_t combined_attrs_fwb(CPUARMState *env, * @s1: Attributes from stage 1 walk * @s2: Attributes from stage 2 walk */ -static ARMCacheAttrs combine_cacheattrs(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) +ARMCacheAttrs combine_cacheattrs(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) { ARMCacheAttrs ret; bool tagged = false; @@ -12848,256 +12829,6 @@ static ARMCacheAttrs combine_cacheattrs(CPUARMState *env, return ret; } - -/* get_phys_addr - get the physical address for this virtual address - * - * Find the physical address corresponding to the given virtual address, - * by doing a translation table walk on MMU based systems or using the - * MPU state on MPU based systems. - * - * Returns false if the translation was successful. Otherwise, phys_ptr, attrs, - * prot and page_size may not be filled in, and the populated fsr value provides - * information on why the translation aborted, in the format of a - * DFSR/IFSR fault register, with the following caveats: - * * we honour the short vs long DFSR format differences. - * * the WnR bit is never set (the caller must do this). - * * for PSMAv5 based systems we don't bother to return a full FSR format - * value. - * - * @env: CPUARMState - * @address: virtual address to get physical address for - * @access_type: 0 for read, 1 for write, 2 for execute - * @mmu_idx: MMU index indicating required translation regime - * @phys_ptr: set to the physical address corresponding to the virtual address - * @attrs: set to the memory transaction attributes to use - * @prot: set to the permissions for the page containing phys_ptr - * @page_size: set to the size of the page containing phys_ptr - * @fi: set to fault info if the translation fails - * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes - */ -bool get_phys_addr(CPUARMState *env, target_ulong address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) -{ - ARMMMUIdx s1_mmu_idx = stage_1_mmu_idx(mmu_idx); - - if (mmu_idx != s1_mmu_idx) { - /* Call ourselves recursively to do the stage 1 and then stage 2 - * translations if mmu_idx is a two-stage regime. - */ - if (arm_feature(env, ARM_FEATURE_EL2)) { - hwaddr ipa; - int s2_prot; - int ret; - bool ipa_secure; - ARMCacheAttrs cacheattrs2 = {}; - ARMMMUIdx s2_mmu_idx; - bool is_el0; - - ret = get_phys_addr(env, address, access_type, s1_mmu_idx, &ipa, - attrs, prot, page_size, fi, cacheattrs); - - /* If S1 fails or S2 is disabled, return early. */ - if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { - *phys_ptr = ipa; - return ret; - } - - ipa_secure = attrs->secure; - if (arm_is_secure_below_el3(env)) { - if (ipa_secure) { - attrs->secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); - } else { - attrs->secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); - } - } else { - assert(!ipa_secure); - } - - s2_mmu_idx = attrs->secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; - is_el0 = mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_SE10_0; - - /* S1 is done. Now do S2 translation. */ - ret = get_phys_addr_lpae(env, ipa, access_type, s2_mmu_idx, is_el0, - phys_ptr, attrs, &s2_prot, - page_size, fi, &cacheattrs2); - fi->s2addr = ipa; - /* Combine the S1 and S2 perms. */ - *prot &= s2_prot; - - /* If S2 fails, return early. */ - if (ret) { - return ret; - } - - /* Combine the S1 and S2 cache attributes. */ - if (arm_hcr_el2_eff(env) & HCR_DC) { - /* - * HCR.DC forces the first stage attributes to - * Normal Non-Shareable, - * Inner Write-Back Read-Allocate Write-Allocate, - * Outer Write-Back Read-Allocate Write-Allocate. - * Do not overwrite Tagged within attrs. - */ - if (cacheattrs->attrs != 0xf0) { - cacheattrs->attrs = 0xff; - } - cacheattrs->shareability = 0; - } - *cacheattrs = combine_cacheattrs(env, *cacheattrs, cacheattrs2); - - /* Check if IPA translates to secure or non-secure PA space. */ - if (arm_is_secure_below_el3(env)) { - if (ipa_secure) { - attrs->secure = - !(env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW)); - } else { - attrs->secure = - !((env->cp15.vtcr_el2.raw_tcr & (VTCR_NSA | VTCR_NSW)) - || (env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW))); - } - } - return 0; - } else { - /* - * For non-EL2 CPUs a stage1+stage2 translation is just stage 1. - */ - mmu_idx = stage_1_mmu_idx(mmu_idx); - } - } - - /* The page table entries may downgrade secure to non-secure, but - * cannot upgrade an non-secure translation regime's attributes - * to secure. - */ - attrs->secure = regime_is_secure(env, mmu_idx); - attrs->user = regime_is_user(env, mmu_idx); - - /* Fast Context Switch Extension. This doesn't exist at all in v8. - * In v7 and earlier it affects all stage 1 translations. - */ - if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 - && !arm_feature(env, ARM_FEATURE_V8)) { - if (regime_el(env, mmu_idx) == 3) { - address += env->cp15.fcseidr_s; - } else { - address += env->cp15.fcseidr_ns; - } - } - - if (arm_feature(env, ARM_FEATURE_PMSA)) { - bool ret; - *page_size = TARGET_PAGE_SIZE; - - if (arm_feature(env, ARM_FEATURE_V8)) { - /* PMSAv8 */ - ret = get_phys_addr_pmsav8(env, address, access_type, mmu_idx, - phys_ptr, attrs, prot, page_size, fi); - } else if (arm_feature(env, ARM_FEATURE_V7)) { - /* PMSAv7 */ - ret = get_phys_addr_pmsav7(env, address, access_type, mmu_idx, - phys_ptr, prot, page_size, fi); - } else { - /* Pre-v7 MPU */ - ret = get_phys_addr_pmsav5(env, address, access_type, mmu_idx, - phys_ptr, prot, fi); - } - qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32 - " mmu_idx %u -> %s (prot %c%c%c)\n", - access_type == MMU_DATA_LOAD ? "reading" : - (access_type == MMU_DATA_STORE ? "writing" : "execute"), - (uint32_t)address, mmu_idx, - ret ? "Miss" : "Hit", - *prot & PAGE_READ ? 'r' : '-', - *prot & PAGE_WRITE ? 'w' : '-', - *prot & PAGE_EXEC ? 'x' : '-'); - - return ret; - } - - /* Definitely a real MMU, not an MPU */ - - if (regime_translation_disabled(env, mmu_idx)) { - uint64_t hcr; - uint8_t memattr; - - /* - * MMU disabled. S1 addresses within aa64 translation regimes are - * still checked for bounds -- see AArch64.TranslateAddressS1Off. - */ - if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { - int r_el = regime_el(env, mmu_idx); - if (arm_el_is_aa64(env, r_el)) { - int pamax = arm_pamax(env_archcpu(env)); - uint64_t tcr = env->cp15.tcr_el[r_el].raw_tcr; - int addrtop, tbi; - - tbi = aa64_va_parameter_tbi(tcr, mmu_idx); - if (access_type == MMU_INST_FETCH) { - tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx); - } - tbi = (tbi >> extract64(address, 55, 1)) & 1; - addrtop = (tbi ? 55 : 63); - - if (extract64(address, pamax, addrtop - pamax + 1) != 0) { - fi->type = ARMFault_AddressSize; - fi->level = 0; - fi->stage2 = false; - return 1; - } - - /* - * When TBI is disabled, we've just validated that all of the - * bits above PAMax are zero, so logically we only need to - * clear the top byte for TBI. But it's clearer to follow - * the pseudocode set of addrdesc.paddress. - */ - address = extract64(address, 0, 52); - } - } - *phys_ptr = address; - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - *page_size = TARGET_PAGE_SIZE; - - /* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */ - hcr = arm_hcr_el2_eff(env); - cacheattrs->shareability = 0; - cacheattrs->is_s2_format = false; - if (hcr & HCR_DC) { - if (hcr & HCR_DCT) { - memattr = 0xf0; /* Tagged, Normal, WB, RWA */ - } else { - memattr = 0xff; /* Normal, WB, RWA */ - } - } else if (access_type == MMU_INST_FETCH) { - if (regime_sctlr(env, mmu_idx) & SCTLR_I) { - memattr = 0xee; /* Normal, WT, RA, NT */ - } else { - memattr = 0x44; /* Normal, NC, No */ - } - cacheattrs->shareability = 2; /* outer sharable */ - } else { - memattr = 0x00; /* Device, nGnRnE */ - } - cacheattrs->attrs = memattr; - return 0; - } - - if (regime_using_lpae_format(env, mmu_idx)) { - return get_phys_addr_lpae(env, address, access_type, mmu_idx, false, - phys_ptr, attrs, prot, page_size, - fi, cacheattrs); - } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) { - return get_phys_addr_v6(env, address, access_type, mmu_idx, - phys_ptr, attrs, prot, page_size, fi); - } else { - return get_phys_addr_v5(env, address, access_type, mmu_idx, - phys_ptr, prot, page_size, fi); - } -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { @@ -13121,7 +12852,6 @@ hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, } return phys_addr; } - #endif /* Note that signed overflow is undefined in C. The following routines are diff --git a/target/arm/internals.h b/target/arm/internals.h index 049edce946..1d83146d56 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -613,8 +613,13 @@ ARMMMUIdx arm_v7m_mmu_idx_for_secstate_and_priv(CPUARMState *env, /* Return the MMU index for a v7M CPU in the specified security state */ ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate); -/* Return true if the stage 1 translation regime is using LPAE format page - * tables */ +/* Return true if the translation regime is using LPAE format page tables */ +bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx); + +/* + * Return true if the stage 1 translation regime is using LPAE + * format page tables + */ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx); /* Raise a data fault alignment exception for the specified virtual address */ @@ -777,6 +782,12 @@ static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) } } +/* Return the SCTLR value which controls this address translation regime */ +static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; +} + /* Return the TCR controlling this translation regime */ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) { @@ -1095,6 +1106,9 @@ typedef struct ARMVAParameters { ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, ARMMMUIdx mmu_idx, bool data); +int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx); +int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx); + static inline int exception_target_el(CPUARMState *env) { int target_el = MAX(1, arm_current_el(env)); diff --git a/target/arm/meson.build b/target/arm/meson.build index 50f152214a..ac571fc45d 100644 --- a/target/arm/meson.build +++ b/target/arm/meson.build @@ -58,6 +58,7 @@ arm_softmmu_ss.add(files( 'machine.c', 'monitor.c', 'psci.c', + 'ptw.c', )) subdir('hvf') diff --git a/target/arm/ptw.c b/target/arm/ptw.c new file mode 100644 index 0000000000..318000f6d9 --- /dev/null +++ b/target/arm/ptw.c @@ -0,0 +1,267 @@ +/* + * ARM page table walking. + * + * This code is licensed under the GNU GPL v2 or later. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "cpu.h" +#include "internals.h" +#include "ptw.h" + + +/** + * get_phys_addr - get the physical address for this virtual address + * + * Find the physical address corresponding to the given virtual address, + * by doing a translation table walk on MMU based systems or using the + * MPU state on MPU based systems. + * + * Returns false if the translation was successful. Otherwise, phys_ptr, attrs, + * prot and page_size may not be filled in, and the populated fsr value provides + * information on why the translation aborted, in the format of a + * DFSR/IFSR fault register, with the following caveats: + * * we honour the short vs long DFSR format differences. + * * the WnR bit is never set (the caller must do this). + * * for PSMAv5 based systems we don't bother to return a full FSR format + * value. + * + * @env: CPUARMState + * @address: virtual address to get physical address for + * @access_type: 0 for read, 1 for write, 2 for execute + * @mmu_idx: MMU index indicating required translation regime + * @phys_ptr: set to the physical address corresponding to the virtual address + * @attrs: set to the memory transaction attributes to use + * @prot: set to the permissions for the page containing phys_ptr + * @page_size: set to the size of the page containing phys_ptr + * @fi: set to fault info if the translation fails + * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes + */ +bool get_phys_addr(CPUARMState *env, target_ulong address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +{ + ARMMMUIdx s1_mmu_idx = stage_1_mmu_idx(mmu_idx); + + if (mmu_idx != s1_mmu_idx) { + /* + * Call ourselves recursively to do the stage 1 and then stage 2 + * translations if mmu_idx is a two-stage regime. + */ + if (arm_feature(env, ARM_FEATURE_EL2)) { + hwaddr ipa; + int s2_prot; + int ret; + bool ipa_secure; + ARMCacheAttrs cacheattrs2 = {}; + ARMMMUIdx s2_mmu_idx; + bool is_el0; + + ret = get_phys_addr(env, address, access_type, s1_mmu_idx, &ipa, + attrs, prot, page_size, fi, cacheattrs); + + /* If S1 fails or S2 is disabled, return early. */ + if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { + *phys_ptr = ipa; + return ret; + } + + ipa_secure = attrs->secure; + if (arm_is_secure_below_el3(env)) { + if (ipa_secure) { + attrs->secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); + } else { + attrs->secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); + } + } else { + assert(!ipa_secure); + } + + s2_mmu_idx = attrs->secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; + is_el0 = mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_SE10_0; + + /* S1 is done. Now do S2 translation. */ + ret = get_phys_addr_lpae(env, ipa, access_type, s2_mmu_idx, is_el0, + phys_ptr, attrs, &s2_prot, + page_size, fi, &cacheattrs2); + fi->s2addr = ipa; + /* Combine the S1 and S2 perms. */ + *prot &= s2_prot; + + /* If S2 fails, return early. */ + if (ret) { + return ret; + } + + /* Combine the S1 and S2 cache attributes. */ + if (arm_hcr_el2_eff(env) & HCR_DC) { + /* + * HCR.DC forces the first stage attributes to + * Normal Non-Shareable, + * Inner Write-Back Read-Allocate Write-Allocate, + * Outer Write-Back Read-Allocate Write-Allocate. + * Do not overwrite Tagged within attrs. + */ + if (cacheattrs->attrs != 0xf0) { + cacheattrs->attrs = 0xff; + } + cacheattrs->shareability = 0; + } + *cacheattrs = combine_cacheattrs(env, *cacheattrs, cacheattrs2); + + /* Check if IPA translates to secure or non-secure PA space. */ + if (arm_is_secure_below_el3(env)) { + if (ipa_secure) { + attrs->secure = + !(env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW)); + } else { + attrs->secure = + !((env->cp15.vtcr_el2.raw_tcr & (VTCR_NSA | VTCR_NSW)) + || (env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW))); + } + } + return 0; + } else { + /* + * For non-EL2 CPUs a stage1+stage2 translation is just stage 1. + */ + mmu_idx = stage_1_mmu_idx(mmu_idx); + } + } + + /* + * The page table entries may downgrade secure to non-secure, but + * cannot upgrade an non-secure translation regime's attributes + * to secure. + */ + attrs->secure = regime_is_secure(env, mmu_idx); + attrs->user = regime_is_user(env, mmu_idx); + + /* + * Fast Context Switch Extension. This doesn't exist at all in v8. + * In v7 and earlier it affects all stage 1 translations. + */ + if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 + && !arm_feature(env, ARM_FEATURE_V8)) { + if (regime_el(env, mmu_idx) == 3) { + address += env->cp15.fcseidr_s; + } else { + address += env->cp15.fcseidr_ns; + } + } + + if (arm_feature(env, ARM_FEATURE_PMSA)) { + bool ret; + *page_size = TARGET_PAGE_SIZE; + + if (arm_feature(env, ARM_FEATURE_V8)) { + /* PMSAv8 */ + ret = get_phys_addr_pmsav8(env, address, access_type, mmu_idx, + phys_ptr, attrs, prot, page_size, fi); + } else if (arm_feature(env, ARM_FEATURE_V7)) { + /* PMSAv7 */ + ret = get_phys_addr_pmsav7(env, address, access_type, mmu_idx, + phys_ptr, prot, page_size, fi); + } else { + /* Pre-v7 MPU */ + ret = get_phys_addr_pmsav5(env, address, access_type, mmu_idx, + phys_ptr, prot, fi); + } + qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32 + " mmu_idx %u -> %s (prot %c%c%c)\n", + access_type == MMU_DATA_LOAD ? "reading" : + (access_type == MMU_DATA_STORE ? "writing" : "execute"), + (uint32_t)address, mmu_idx, + ret ? "Miss" : "Hit", + *prot & PAGE_READ ? 'r' : '-', + *prot & PAGE_WRITE ? 'w' : '-', + *prot & PAGE_EXEC ? 'x' : '-'); + + return ret; + } + + /* Definitely a real MMU, not an MPU */ + + if (regime_translation_disabled(env, mmu_idx)) { + uint64_t hcr; + uint8_t memattr; + + /* + * MMU disabled. S1 addresses within aa64 translation regimes are + * still checked for bounds -- see AArch64.TranslateAddressS1Off. + */ + if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { + int r_el = regime_el(env, mmu_idx); + if (arm_el_is_aa64(env, r_el)) { + int pamax = arm_pamax(env_archcpu(env)); + uint64_t tcr = env->cp15.tcr_el[r_el].raw_tcr; + int addrtop, tbi; + + tbi = aa64_va_parameter_tbi(tcr, mmu_idx); + if (access_type == MMU_INST_FETCH) { + tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx); + } + tbi = (tbi >> extract64(address, 55, 1)) & 1; + addrtop = (tbi ? 55 : 63); + + if (extract64(address, pamax, addrtop - pamax + 1) != 0) { + fi->type = ARMFault_AddressSize; + fi->level = 0; + fi->stage2 = false; + return 1; + } + + /* + * When TBI is disabled, we've just validated that all of the + * bits above PAMax are zero, so logically we only need to + * clear the top byte for TBI. But it's clearer to follow + * the pseudocode set of addrdesc.paddress. + */ + address = extract64(address, 0, 52); + } + } + *phys_ptr = address; + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + *page_size = TARGET_PAGE_SIZE; + + /* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */ + hcr = arm_hcr_el2_eff(env); + cacheattrs->shareability = 0; + cacheattrs->is_s2_format = false; + if (hcr & HCR_DC) { + if (hcr & HCR_DCT) { + memattr = 0xf0; /* Tagged, Normal, WB, RWA */ + } else { + memattr = 0xff; /* Normal, WB, RWA */ + } + } else if (access_type == MMU_INST_FETCH) { + if (regime_sctlr(env, mmu_idx) & SCTLR_I) { + memattr = 0xee; /* Normal, WT, RA, NT */ + } else { + memattr = 0x44; /* Normal, NC, No */ + } + cacheattrs->shareability = 2; /* outer sharable */ + } else { + memattr = 0x00; /* Device, nGnRnE */ + } + cacheattrs->attrs = memattr; + return 0; + } + + if (regime_using_lpae_format(env, mmu_idx)) { + return get_phys_addr_lpae(env, address, access_type, mmu_idx, false, + phys_ptr, attrs, prot, page_size, + fi, cacheattrs); + } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) { + return get_phys_addr_v6(env, address, access_type, mmu_idx, + phys_ptr, attrs, prot, page_size, fi); + } else { + return get_phys_addr_v5(env, address, access_type, mmu_idx, + phys_ptr, prot, page_size, fi); + } +} diff --git a/target/arm/ptw.h b/target/arm/ptw.h new file mode 100644 index 0000000000..e2023ae750 --- /dev/null +++ b/target/arm/ptw.h @@ -0,0 +1,51 @@ +/* + * ARM page table walking. + * + * This code is licensed under the GNU GPL v2 or later. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef TARGET_ARM_PTW_H +#define TARGET_ARM_PTW_H + +#ifndef CONFIG_USER_ONLY + +bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); +bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); +ARMCacheAttrs combine_cacheattrs(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2); + +bool get_phys_addr_v5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi); +bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + ARMMMUFaultInfo *fi); +bool get_phys_addr_v6(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, ARMMMUFaultInfo *fi); +bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi); +bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, target_ulong *page_size, + ARMMMUFaultInfo *fi); +bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, + hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) + __attribute__((nonnull)); + +#endif /* !CONFIG_USER_ONLY */ +#endif /* TARGET_ARM_PTW_H */ From f2d2f5ceb87e828546f7b544f1289193f3b920ce Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 10/55] target/arm: Move get_phys_addr_v5 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-4-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 137 +++----------------------------------------- target/arm/ptw.c | 123 +++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 15 +++-- 3 files changed, 140 insertions(+), 135 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 3ffd122178..321716914b 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10578,8 +10578,7 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) * @ap: The 3-bit access permissions (AP[2:0]) * @domain_prot: The 2-bit domain access permissions */ -static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, - int ap, int domain_prot) +int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot) { bool is_user = regime_is_user(env, mmu_idx); @@ -10782,8 +10781,8 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } -static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, - uint32_t *table, uint32_t address) +bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, + uint32_t *table, uint32_t address) { /* Note that we can only get here for an AArch32 PL0/PL1 lookup */ TCR *tcr = regime_tcr(env, mmu_idx); @@ -10882,8 +10881,8 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, } /* All loads done in the course of a page table walk go through here. */ -static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { ARMCPU *cpu = ARM_CPU(cs); CPUARMState *env = &cpu->env; @@ -10911,8 +10910,8 @@ static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { ARMCPU *cpu = ARM_CPU(cs); CPUARMState *env = &cpu->env; @@ -10940,128 +10939,6 @@ static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -bool get_phys_addr_v5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) -{ - CPUState *cs = env_cpu(env); - int level = 1; - uint32_t table; - uint32_t desc; - int type; - int ap; - int domain = 0; - int domain_prot; - hwaddr phys_addr; - uint32_t dacr; - - /* Pagetable walk. */ - /* Lookup l1 descriptor. */ - if (!get_level1_table_address(env, mmu_idx, &table, address)) { - /* Section translation fault if page walk is disabled by PD0 or PD1 */ - fi->type = ARMFault_Translation; - goto do_fault; - } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - type = (desc & 3); - domain = (desc >> 5) & 0x0f; - if (regime_el(env, mmu_idx) == 1) { - dacr = env->cp15.dacr_ns; - } else { - dacr = env->cp15.dacr_s; - } - domain_prot = (dacr >> (domain * 2)) & 3; - if (type == 0) { - /* Section translation fault. */ - fi->type = ARMFault_Translation; - goto do_fault; - } - if (type != 2) { - level = 2; - } - if (domain_prot == 0 || domain_prot == 2) { - fi->type = ARMFault_Domain; - goto do_fault; - } - if (type == 2) { - /* 1Mb section. */ - phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); - ap = (desc >> 10) & 3; - *page_size = 1024 * 1024; - } else { - /* Lookup l2 entry. */ - if (type == 1) { - /* Coarse pagetable. */ - table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); - } else { - /* Fine pagetable. */ - table = (desc & 0xfffff000) | ((address >> 8) & 0xffc); - } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - switch (desc & 3) { - case 0: /* Page translation fault. */ - fi->type = ARMFault_Translation; - goto do_fault; - case 1: /* 64k page. */ - phys_addr = (desc & 0xffff0000) | (address & 0xffff); - ap = (desc >> (4 + ((address >> 13) & 6))) & 3; - *page_size = 0x10000; - break; - case 2: /* 4k page. */ - phys_addr = (desc & 0xfffff000) | (address & 0xfff); - ap = (desc >> (4 + ((address >> 9) & 6))) & 3; - *page_size = 0x1000; - break; - case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */ - if (type == 1) { - /* ARMv6/XScale extended small page format */ - if (arm_feature(env, ARM_FEATURE_XSCALE) - || arm_feature(env, ARM_FEATURE_V6)) { - phys_addr = (desc & 0xfffff000) | (address & 0xfff); - *page_size = 0x1000; - } else { - /* UNPREDICTABLE in ARMv5; we choose to take a - * page translation fault. - */ - fi->type = ARMFault_Translation; - goto do_fault; - } - } else { - phys_addr = (desc & 0xfffffc00) | (address & 0x3ff); - *page_size = 0x400; - } - ap = (desc >> 4) & 3; - break; - default: - /* Never happens, but compiler isn't smart enough to tell. */ - g_assert_not_reached(); - } - } - *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); - *prot |= *prot ? PAGE_EXEC : 0; - if (!(*prot & (1 << access_type))) { - /* Access permission fault. */ - fi->type = ARMFault_Permission; - goto do_fault; - } - *phys_ptr = phys_addr; - return false; -do_fault: - fi->domain = domain; - fi->level = level; - return true; -} - bool get_phys_addr_v6(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 318000f6d9..09c4472628 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -13,6 +13,129 @@ #include "ptw.h" +static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) +{ + CPUState *cs = env_cpu(env); + int level = 1; + uint32_t table; + uint32_t desc; + int type; + int ap; + int domain = 0; + int domain_prot; + hwaddr phys_addr; + uint32_t dacr; + + /* Pagetable walk. */ + /* Lookup l1 descriptor. */ + if (!get_level1_table_address(env, mmu_idx, &table, address)) { + /* Section translation fault if page walk is disabled by PD0 or PD1 */ + fi->type = ARMFault_Translation; + goto do_fault; + } + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + type = (desc & 3); + domain = (desc >> 5) & 0x0f; + if (regime_el(env, mmu_idx) == 1) { + dacr = env->cp15.dacr_ns; + } else { + dacr = env->cp15.dacr_s; + } + domain_prot = (dacr >> (domain * 2)) & 3; + if (type == 0) { + /* Section translation fault. */ + fi->type = ARMFault_Translation; + goto do_fault; + } + if (type != 2) { + level = 2; + } + if (domain_prot == 0 || domain_prot == 2) { + fi->type = ARMFault_Domain; + goto do_fault; + } + if (type == 2) { + /* 1Mb section. */ + phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); + ap = (desc >> 10) & 3; + *page_size = 1024 * 1024; + } else { + /* Lookup l2 entry. */ + if (type == 1) { + /* Coarse pagetable. */ + table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); + } else { + /* Fine pagetable. */ + table = (desc & 0xfffff000) | ((address >> 8) & 0xffc); + } + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + switch (desc & 3) { + case 0: /* Page translation fault. */ + fi->type = ARMFault_Translation; + goto do_fault; + case 1: /* 64k page. */ + phys_addr = (desc & 0xffff0000) | (address & 0xffff); + ap = (desc >> (4 + ((address >> 13) & 6))) & 3; + *page_size = 0x10000; + break; + case 2: /* 4k page. */ + phys_addr = (desc & 0xfffff000) | (address & 0xfff); + ap = (desc >> (4 + ((address >> 9) & 6))) & 3; + *page_size = 0x1000; + break; + case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */ + if (type == 1) { + /* ARMv6/XScale extended small page format */ + if (arm_feature(env, ARM_FEATURE_XSCALE) + || arm_feature(env, ARM_FEATURE_V6)) { + phys_addr = (desc & 0xfffff000) | (address & 0xfff); + *page_size = 0x1000; + } else { + /* + * UNPREDICTABLE in ARMv5; we choose to take a + * page translation fault. + */ + fi->type = ARMFault_Translation; + goto do_fault; + } + } else { + phys_addr = (desc & 0xfffffc00) | (address & 0x3ff); + *page_size = 0x400; + } + ap = (desc >> 4) & 3; + break; + default: + /* Never happens, but compiler isn't smart enough to tell. */ + g_assert_not_reached(); + } + } + *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); + *prot |= *prot ? PAGE_EXEC : 0; + if (!(*prot & (1 << access_type))) { + /* Access permission fault. */ + fi->type = ARMFault_Permission; + goto do_fault; + } + *phys_ptr = phys_addr; + return false; +do_fault: + fi->domain = domain; + fi->level = level; + return true; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index e2023ae750..2dbd97b8cb 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,16 +11,21 @@ #ifndef CONFIG_USER_ONLY +uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); +uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); + bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); ARMCacheAttrs combine_cacheattrs(CPUARMState *env, ARMCacheAttrs s1, ARMCacheAttrs s2); -bool get_phys_addr_v5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi); +bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, + uint32_t *table, uint32_t address); +int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, + int ap, int domain_prot); + bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 53c038efb7a634dd5b6ede8eb034791c0dc83f43 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 11/55] target/arm: Move get_phys_addr_v6 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-5-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 161 +------------------------------------------- target/arm/ptw.c | 153 +++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 11 +-- 3 files changed, 161 insertions(+), 164 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 321716914b..4a58822025 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10631,7 +10631,7 @@ int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot) * @ap: The 2-bit simple AP (AP[2:1]) * @is_user: TRUE if accessing from PL0 */ -static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user) +int simple_ap_to_rw_prot_is_user(int ap, bool is_user) { switch (ap) { case 0: @@ -10647,12 +10647,6 @@ static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user) } } -static inline int -simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) -{ - return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); -} - /* Translate S2 section/page access permissions to protection flags * * @env: CPUARMState @@ -10939,159 +10933,6 @@ uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -bool get_phys_addr_v6(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, ARMMMUFaultInfo *fi) -{ - CPUState *cs = env_cpu(env); - ARMCPU *cpu = env_archcpu(env); - int level = 1; - uint32_t table; - uint32_t desc; - uint32_t xn; - uint32_t pxn = 0; - int type; - int ap; - int domain = 0; - int domain_prot; - hwaddr phys_addr; - uint32_t dacr; - bool ns; - - /* Pagetable walk. */ - /* Lookup l1 descriptor. */ - if (!get_level1_table_address(env, mmu_idx, &table, address)) { - /* Section translation fault if page walk is disabled by PD0 or PD1 */ - fi->type = ARMFault_Translation; - goto do_fault; - } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - type = (desc & 3); - if (type == 0 || (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) { - /* Section translation fault, or attempt to use the encoding - * which is Reserved on implementations without PXN. - */ - fi->type = ARMFault_Translation; - goto do_fault; - } - if ((type == 1) || !(desc & (1 << 18))) { - /* Page or Section. */ - domain = (desc >> 5) & 0x0f; - } - if (regime_el(env, mmu_idx) == 1) { - dacr = env->cp15.dacr_ns; - } else { - dacr = env->cp15.dacr_s; - } - if (type == 1) { - level = 2; - } - domain_prot = (dacr >> (domain * 2)) & 3; - if (domain_prot == 0 || domain_prot == 2) { - /* Section or Page domain fault */ - fi->type = ARMFault_Domain; - goto do_fault; - } - if (type != 1) { - if (desc & (1 << 18)) { - /* Supersection. */ - phys_addr = (desc & 0xff000000) | (address & 0x00ffffff); - phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32; - phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36; - *page_size = 0x1000000; - } else { - /* Section. */ - phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); - *page_size = 0x100000; - } - ap = ((desc >> 10) & 3) | ((desc >> 13) & 4); - xn = desc & (1 << 4); - pxn = desc & 1; - ns = extract32(desc, 19, 1); - } else { - if (cpu_isar_feature(aa32_pxn, cpu)) { - pxn = (desc >> 2) & 1; - } - ns = extract32(desc, 3, 1); - /* Lookup l2 entry. */ - table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - ap = ((desc >> 4) & 3) | ((desc >> 7) & 4); - switch (desc & 3) { - case 0: /* Page translation fault. */ - fi->type = ARMFault_Translation; - goto do_fault; - case 1: /* 64k page. */ - phys_addr = (desc & 0xffff0000) | (address & 0xffff); - xn = desc & (1 << 15); - *page_size = 0x10000; - break; - case 2: case 3: /* 4k page. */ - phys_addr = (desc & 0xfffff000) | (address & 0xfff); - xn = desc & 1; - *page_size = 0x1000; - break; - default: - /* Never happens, but compiler isn't smart enough to tell. */ - g_assert_not_reached(); - } - } - if (domain_prot == 3) { - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - } else { - if (pxn && !regime_is_user(env, mmu_idx)) { - xn = 1; - } - if (xn && access_type == MMU_INST_FETCH) { - fi->type = ARMFault_Permission; - goto do_fault; - } - - if (arm_feature(env, ARM_FEATURE_V6K) && - (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) { - /* The simplified model uses AP[0] as an access control bit. */ - if ((ap & 1) == 0) { - /* Access flag fault. */ - fi->type = ARMFault_AccessFlag; - goto do_fault; - } - *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1); - } else { - *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); - } - if (*prot && !xn) { - *prot |= PAGE_EXEC; - } - if (!(*prot & (1 << access_type))) { - /* Access permission fault. */ - fi->type = ARMFault_Permission; - goto do_fault; - } - } - if (ns) { - /* The NS bit will (as required by the architecture) have no effect if - * the CPU doesn't support TZ or this is a non-secure translation - * regime, because the attribute will already be non-secure. - */ - attrs->secure = false; - } - *phys_ptr = phys_addr; - return false; -do_fault: - fi->domain = domain; - fi->level = level; - return true; -} - /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 09c4472628..6a1f4b549d 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -136,6 +136,159 @@ do_fault: return true; } +static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, ARMMMUFaultInfo *fi) +{ + CPUState *cs = env_cpu(env); + ARMCPU *cpu = env_archcpu(env); + int level = 1; + uint32_t table; + uint32_t desc; + uint32_t xn; + uint32_t pxn = 0; + int type; + int ap; + int domain = 0; + int domain_prot; + hwaddr phys_addr; + uint32_t dacr; + bool ns; + + /* Pagetable walk. */ + /* Lookup l1 descriptor. */ + if (!get_level1_table_address(env, mmu_idx, &table, address)) { + /* Section translation fault if page walk is disabled by PD0 or PD1 */ + fi->type = ARMFault_Translation; + goto do_fault; + } + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + type = (desc & 3); + if (type == 0 || (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) { + /* Section translation fault, or attempt to use the encoding + * which is Reserved on implementations without PXN. + */ + fi->type = ARMFault_Translation; + goto do_fault; + } + if ((type == 1) || !(desc & (1 << 18))) { + /* Page or Section. */ + domain = (desc >> 5) & 0x0f; + } + if (regime_el(env, mmu_idx) == 1) { + dacr = env->cp15.dacr_ns; + } else { + dacr = env->cp15.dacr_s; + } + if (type == 1) { + level = 2; + } + domain_prot = (dacr >> (domain * 2)) & 3; + if (domain_prot == 0 || domain_prot == 2) { + /* Section or Page domain fault */ + fi->type = ARMFault_Domain; + goto do_fault; + } + if (type != 1) { + if (desc & (1 << 18)) { + /* Supersection. */ + phys_addr = (desc & 0xff000000) | (address & 0x00ffffff); + phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32; + phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36; + *page_size = 0x1000000; + } else { + /* Section. */ + phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); + *page_size = 0x100000; + } + ap = ((desc >> 10) & 3) | ((desc >> 13) & 4); + xn = desc & (1 << 4); + pxn = desc & 1; + ns = extract32(desc, 19, 1); + } else { + if (cpu_isar_feature(aa32_pxn, cpu)) { + pxn = (desc >> 2) & 1; + } + ns = extract32(desc, 3, 1); + /* Lookup l2 entry. */ + table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + ap = ((desc >> 4) & 3) | ((desc >> 7) & 4); + switch (desc & 3) { + case 0: /* Page translation fault. */ + fi->type = ARMFault_Translation; + goto do_fault; + case 1: /* 64k page. */ + phys_addr = (desc & 0xffff0000) | (address & 0xffff); + xn = desc & (1 << 15); + *page_size = 0x10000; + break; + case 2: case 3: /* 4k page. */ + phys_addr = (desc & 0xfffff000) | (address & 0xfff); + xn = desc & 1; + *page_size = 0x1000; + break; + default: + /* Never happens, but compiler isn't smart enough to tell. */ + g_assert_not_reached(); + } + } + if (domain_prot == 3) { + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + } else { + if (pxn && !regime_is_user(env, mmu_idx)) { + xn = 1; + } + if (xn && access_type == MMU_INST_FETCH) { + fi->type = ARMFault_Permission; + goto do_fault; + } + + if (arm_feature(env, ARM_FEATURE_V6K) && + (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) { + /* The simplified model uses AP[0] as an access control bit. */ + if ((ap & 1) == 0) { + /* Access flag fault. */ + fi->type = ARMFault_AccessFlag; + goto do_fault; + } + *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1); + } else { + *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); + } + if (*prot && !xn) { + *prot |= PAGE_EXEC; + } + if (!(*prot & (1 << access_type))) { + /* Access permission fault. */ + fi->type = ARMFault_Permission; + goto do_fault; + } + } + if (ns) { + /* The NS bit will (as required by the architecture) have no effect if + * the CPU doesn't support TZ or this is a non-secure translation + * regime, because the attribute will already be non-secure. + */ + attrs->secure = false; + } + *phys_ptr = phys_addr; + return false; +do_fault: + fi->domain = domain; + fi->level = level; + return true; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 2dbd97b8cb..349b842d3c 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -25,15 +25,18 @@ bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, uint32_t *table, uint32_t address); int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot); +int simple_ap_to_rw_prot_is_user(int ap, bool is_user); + +static inline int +simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) +{ + return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); +} bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, ARMMMUFaultInfo *fi); -bool get_phys_addr_v6(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, ARMMMUFaultInfo *fi); bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 9a12fb366d442751603dc91ab5c5f9b7c828c783 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 12/55] target/arm: Move get_phys_addr_pmsav5 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-6-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 85 --------------------------------------------- target/arm/ptw.c | 85 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 4 --- 3 files changed, 85 insertions(+), 89 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 4a58822025..5d01019010 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -12274,91 +12274,6 @@ bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, return ret; } -bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - ARMMMUFaultInfo *fi) -{ - int n; - uint32_t mask; - uint32_t base; - bool is_user = regime_is_user(env, mmu_idx); - - if (regime_translation_disabled(env, mmu_idx)) { - /* MPU disabled. */ - *phys_ptr = address; - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - return false; - } - - *phys_ptr = address; - for (n = 7; n >= 0; n--) { - base = env->cp15.c6_region[n]; - if ((base & 1) == 0) { - continue; - } - mask = 1 << ((base >> 1) & 0x1f); - /* Keep this shift separate from the above to avoid an - (undefined) << 32. */ - mask = (mask << 1) - 1; - if (((base ^ address) & ~mask) == 0) { - break; - } - } - if (n < 0) { - fi->type = ARMFault_Background; - return true; - } - - if (access_type == MMU_INST_FETCH) { - mask = env->cp15.pmsav5_insn_ap; - } else { - mask = env->cp15.pmsav5_data_ap; - } - mask = (mask >> (n * 4)) & 0xf; - switch (mask) { - case 0: - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - case 1: - if (is_user) { - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - *prot = PAGE_READ | PAGE_WRITE; - break; - case 2: - *prot = PAGE_READ; - if (!is_user) { - *prot |= PAGE_WRITE; - } - break; - case 3: - *prot = PAGE_READ | PAGE_WRITE; - break; - case 5: - if (is_user) { - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - *prot = PAGE_READ; - break; - case 6: - *prot = PAGE_READ; - break; - default: - /* Bad permission. */ - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - *prot |= PAGE_EXEC; - return false; -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 6a1f4b549d..5c32648a16 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -289,6 +289,91 @@ do_fault: return true; } +static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + ARMMMUFaultInfo *fi) +{ + int n; + uint32_t mask; + uint32_t base; + bool is_user = regime_is_user(env, mmu_idx); + + if (regime_translation_disabled(env, mmu_idx)) { + /* MPU disabled. */ + *phys_ptr = address; + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + return false; + } + + *phys_ptr = address; + for (n = 7; n >= 0; n--) { + base = env->cp15.c6_region[n]; + if ((base & 1) == 0) { + continue; + } + mask = 1 << ((base >> 1) & 0x1f); + /* Keep this shift separate from the above to avoid an + (undefined) << 32. */ + mask = (mask << 1) - 1; + if (((base ^ address) & ~mask) == 0) { + break; + } + } + if (n < 0) { + fi->type = ARMFault_Background; + return true; + } + + if (access_type == MMU_INST_FETCH) { + mask = env->cp15.pmsav5_insn_ap; + } else { + mask = env->cp15.pmsav5_data_ap; + } + mask = (mask >> (n * 4)) & 0xf; + switch (mask) { + case 0: + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + case 1: + if (is_user) { + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + *prot = PAGE_READ | PAGE_WRITE; + break; + case 2: + *prot = PAGE_READ; + if (!is_user) { + *prot |= PAGE_WRITE; + } + break; + case 3: + *prot = PAGE_READ | PAGE_WRITE; + break; + case 5: + if (is_user) { + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + *prot = PAGE_READ; + break; + case 6: + *prot = PAGE_READ; + break; + default: + /* Bad permission. */ + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + *prot |= PAGE_EXEC; + return false; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 349b842d3c..324a9dde14 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,10 +33,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } -bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - ARMMMUFaultInfo *fi); bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 7d2e08c96077bcf7ed31b6b7c7b7066801b4b89d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 13/55] target/arm: Move get_phys_addr_pmsav7_default to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-7-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 41 ----------------------------------------- target/arm/ptw.c | 41 +++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 +++ 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 5d01019010..d4f7c05625 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11678,47 +11678,6 @@ do_fault: return true; } -static inline void get_phys_addr_pmsav7_default(CPUARMState *env, - ARMMMUIdx mmu_idx, - int32_t address, int *prot) -{ - if (!arm_feature(env, ARM_FEATURE_M)) { - *prot = PAGE_READ | PAGE_WRITE; - switch (address) { - case 0xF0000000 ... 0xFFFFFFFF: - if (regime_sctlr(env, mmu_idx) & SCTLR_V) { - /* hivecs execing is ok */ - *prot |= PAGE_EXEC; - } - break; - case 0x00000000 ... 0x7FFFFFFF: - *prot |= PAGE_EXEC; - break; - } - } else { - /* Default system address map for M profile cores. - * The architecture specifies which regions are execute-never; - * at the MPU level no other checks are defined. - */ - switch (address) { - case 0x00000000 ... 0x1fffffff: /* ROM */ - case 0x20000000 ... 0x3fffffff: /* SRAM */ - case 0x60000000 ... 0x7fffffff: /* RAM */ - case 0x80000000 ... 0x9fffffff: /* RAM */ - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - break; - case 0x40000000 ... 0x5fffffff: /* Peripheral */ - case 0xa0000000 ... 0xbfffffff: /* Device */ - case 0xc0000000 ... 0xdfffffff: /* Device */ - case 0xe0000000 ... 0xffffffff: /* System */ - *prot = PAGE_READ | PAGE_WRITE; - break; - default: - g_assert_not_reached(); - } - } -} - static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 5c32648a16..74650c6c52 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -374,6 +374,47 @@ static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, return false; } +void get_phys_addr_pmsav7_default(CPUARMState *env, + ARMMMUIdx mmu_idx, + int32_t address, int *prot) +{ + if (!arm_feature(env, ARM_FEATURE_M)) { + *prot = PAGE_READ | PAGE_WRITE; + switch (address) { + case 0xF0000000 ... 0xFFFFFFFF: + if (regime_sctlr(env, mmu_idx) & SCTLR_V) { + /* hivecs execing is ok */ + *prot |= PAGE_EXEC; + } + break; + case 0x00000000 ... 0x7FFFFFFF: + *prot |= PAGE_EXEC; + break; + } + } else { + /* Default system address map for M profile cores. + * The architecture specifies which regions are execute-never; + * at the MPU level no other checks are defined. + */ + switch (address) { + case 0x00000000 ... 0x1fffffff: /* ROM */ + case 0x20000000 ... 0x3fffffff: /* SRAM */ + case 0x60000000 ... 0x7fffffff: /* RAM */ + case 0x80000000 ... 0x9fffffff: /* RAM */ + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + break; + case 0x40000000 ... 0x5fffffff: /* Peripheral */ + case 0xa0000000 ... 0xbfffffff: /* Device */ + case 0xc0000000 ... 0xdfffffff: /* Device */ + case 0xe0000000 ... 0xffffffff: /* System */ + *prot = PAGE_READ | PAGE_WRITE; + break; + default: + g_assert_not_reached(); + } + } +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 324a9dde14..d6e3fee152 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,6 +33,9 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } +void get_phys_addr_pmsav7_default(CPUARMState *env, + ARMMMUIdx mmu_idx, + int32_t address, int *prot); bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 1f2e87e5aba133ecf4c85000a61d5d760176dd6c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 14/55] target/arm: Move get_phys_addr_pmsav7 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-8-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 194 +------------------------------------------- target/arm/ptw.c | 190 +++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 10 +-- 3 files changed, 198 insertions(+), 196 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d4f7c05625..2ebaf69407 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11678,8 +11678,7 @@ do_fault: return true; } -static bool pmsav7_use_background_region(ARMCPU *cpu, - ARMMMUIdx mmu_idx, bool is_user) +bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) { /* Return true if we should use the default memory map as a * "background" region if there are no hits against any MPU regions. @@ -11698,14 +11697,14 @@ static bool pmsav7_use_background_region(ARMCPU *cpu, } } -static inline bool m_is_ppb_region(CPUARMState *env, uint32_t address) +bool m_is_ppb_region(CPUARMState *env, uint32_t address) { /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ return arm_feature(env, ARM_FEATURE_M) && extract32(address, 20, 12) == 0xe00; } -static inline bool m_is_system_region(CPUARMState *env, uint32_t address) +bool m_is_system_region(CPUARMState *env, uint32_t address) { /* True if address is in the M profile system region * 0xe0000000 - 0xffffffff @@ -11713,193 +11712,6 @@ static inline bool m_is_system_region(CPUARMState *env, uint32_t address) return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; } -bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) -{ - ARMCPU *cpu = env_archcpu(env); - int n; - bool is_user = regime_is_user(env, mmu_idx); - - *phys_ptr = address; - *page_size = TARGET_PAGE_SIZE; - *prot = 0; - - if (regime_translation_disabled(env, mmu_idx) || - m_is_ppb_region(env, address)) { - /* MPU disabled or M profile PPB access: use default memory map. - * The other case which uses the default memory map in the - * v7M ARM ARM pseudocode is exception vector reads from the vector - * table. In QEMU those accesses are done in arm_v7m_load_vector(), - * which always does a direct read using address_space_ldl(), rather - * than going via this function, so we don't need to check that here. - */ - get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); - } else { /* MPU enabled */ - for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { - /* region search */ - uint32_t base = env->pmsav7.drbar[n]; - uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5); - uint32_t rmask; - bool srdis = false; - - if (!(env->pmsav7.drsr[n] & 0x1)) { - continue; - } - - if (!rsize) { - qemu_log_mask(LOG_GUEST_ERROR, - "DRSR[%d]: Rsize field cannot be 0\n", n); - continue; - } - rsize++; - rmask = (1ull << rsize) - 1; - - if (base & rmask) { - qemu_log_mask(LOG_GUEST_ERROR, - "DRBAR[%d]: 0x%" PRIx32 " misaligned " - "to DRSR region size, mask = 0x%" PRIx32 "\n", - n, base, rmask); - continue; - } - - if (address < base || address > base + rmask) { - /* - * Address not in this region. We must check whether the - * region covers addresses in the same page as our address. - * In that case we must not report a size that covers the - * whole page for a subsequent hit against a different MPU - * region or the background region, because it would result in - * incorrect TLB hits for subsequent accesses to addresses that - * are in this MPU region. - */ - if (ranges_overlap(base, rmask, - address & TARGET_PAGE_MASK, - TARGET_PAGE_SIZE)) { - *page_size = 1; - } - continue; - } - - /* Region matched */ - - if (rsize >= 8) { /* no subregions for regions < 256 bytes */ - int i, snd; - uint32_t srdis_mask; - - rsize -= 3; /* sub region size (power of 2) */ - snd = ((address - base) >> rsize) & 0x7; - srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1); - - srdis_mask = srdis ? 0x3 : 0x0; - for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) { - /* This will check in groups of 2, 4 and then 8, whether - * the subregion bits are consistent. rsize is incremented - * back up to give the region size, considering consistent - * adjacent subregions as one region. Stop testing if rsize - * is already big enough for an entire QEMU page. - */ - int snd_rounded = snd & ~(i - 1); - uint32_t srdis_multi = extract32(env->pmsav7.drsr[n], - snd_rounded + 8, i); - if (srdis_mask ^ srdis_multi) { - break; - } - srdis_mask = (srdis_mask << i) | srdis_mask; - rsize++; - } - } - if (srdis) { - continue; - } - if (rsize < TARGET_PAGE_BITS) { - *page_size = 1 << rsize; - } - break; - } - - if (n == -1) { /* no hits */ - if (!pmsav7_use_background_region(cpu, mmu_idx, is_user)) { - /* background fault */ - fi->type = ARMFault_Background; - return true; - } - get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); - } else { /* a MPU hit! */ - uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3); - uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1); - - if (m_is_system_region(env, address)) { - /* System space is always execute never */ - xn = 1; - } - - if (is_user) { /* User mode AP bit decoding */ - switch (ap) { - case 0: - case 1: - case 5: - break; /* no access */ - case 3: - *prot |= PAGE_WRITE; - /* fall through */ - case 2: - case 6: - *prot |= PAGE_READ | PAGE_EXEC; - break; - case 7: - /* for v7M, same as 6; for R profile a reserved value */ - if (arm_feature(env, ARM_FEATURE_M)) { - *prot |= PAGE_READ | PAGE_EXEC; - break; - } - /* fall through */ - default: - qemu_log_mask(LOG_GUEST_ERROR, - "DRACR[%d]: Bad value for AP bits: 0x%" - PRIx32 "\n", n, ap); - } - } else { /* Priv. mode AP bits decoding */ - switch (ap) { - case 0: - break; /* no access */ - case 1: - case 2: - case 3: - *prot |= PAGE_WRITE; - /* fall through */ - case 5: - case 6: - *prot |= PAGE_READ | PAGE_EXEC; - break; - case 7: - /* for v7M, same as 6; for R profile a reserved value */ - if (arm_feature(env, ARM_FEATURE_M)) { - *prot |= PAGE_READ | PAGE_EXEC; - break; - } - /* fall through */ - default: - qemu_log_mask(LOG_GUEST_ERROR, - "DRACR[%d]: Bad value for AP bits: 0x%" - PRIx32 "\n", n, ap); - } - } - - /* execute never */ - if (xn) { - *prot &= ~PAGE_EXEC; - } - } - } - - fi->type = ARMFault_Permission; - fi->level = 1; - return !(*prot & (1 << access_type)); -} - static bool v8m_is_sau_exempt(CPUARMState *env, uint32_t address, MMUAccessType access_type) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 74650c6c52..27715dbfa8 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -8,6 +8,7 @@ #include "qemu/osdep.h" #include "qemu/log.h" +#include "qemu/range.h" #include "cpu.h" #include "internals.h" #include "ptw.h" @@ -415,6 +416,195 @@ void get_phys_addr_pmsav7_default(CPUARMState *env, } } +static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) +{ + ARMCPU *cpu = env_archcpu(env); + int n; + bool is_user = regime_is_user(env, mmu_idx); + + *phys_ptr = address; + *page_size = TARGET_PAGE_SIZE; + *prot = 0; + + if (regime_translation_disabled(env, mmu_idx) || + m_is_ppb_region(env, address)) { + /* + * MPU disabled or M profile PPB access: use default memory map. + * The other case which uses the default memory map in the + * v7M ARM ARM pseudocode is exception vector reads from the vector + * table. In QEMU those accesses are done in arm_v7m_load_vector(), + * which always does a direct read using address_space_ldl(), rather + * than going via this function, so we don't need to check that here. + */ + get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); + } else { /* MPU enabled */ + for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { + /* region search */ + uint32_t base = env->pmsav7.drbar[n]; + uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5); + uint32_t rmask; + bool srdis = false; + + if (!(env->pmsav7.drsr[n] & 0x1)) { + continue; + } + + if (!rsize) { + qemu_log_mask(LOG_GUEST_ERROR, + "DRSR[%d]: Rsize field cannot be 0\n", n); + continue; + } + rsize++; + rmask = (1ull << rsize) - 1; + + if (base & rmask) { + qemu_log_mask(LOG_GUEST_ERROR, + "DRBAR[%d]: 0x%" PRIx32 " misaligned " + "to DRSR region size, mask = 0x%" PRIx32 "\n", + n, base, rmask); + continue; + } + + if (address < base || address > base + rmask) { + /* + * Address not in this region. We must check whether the + * region covers addresses in the same page as our address. + * In that case we must not report a size that covers the + * whole page for a subsequent hit against a different MPU + * region or the background region, because it would result in + * incorrect TLB hits for subsequent accesses to addresses that + * are in this MPU region. + */ + if (ranges_overlap(base, rmask, + address & TARGET_PAGE_MASK, + TARGET_PAGE_SIZE)) { + *page_size = 1; + } + continue; + } + + /* Region matched */ + + if (rsize >= 8) { /* no subregions for regions < 256 bytes */ + int i, snd; + uint32_t srdis_mask; + + rsize -= 3; /* sub region size (power of 2) */ + snd = ((address - base) >> rsize) & 0x7; + srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1); + + srdis_mask = srdis ? 0x3 : 0x0; + for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) { + /* + * This will check in groups of 2, 4 and then 8, whether + * the subregion bits are consistent. rsize is incremented + * back up to give the region size, considering consistent + * adjacent subregions as one region. Stop testing if rsize + * is already big enough for an entire QEMU page. + */ + int snd_rounded = snd & ~(i - 1); + uint32_t srdis_multi = extract32(env->pmsav7.drsr[n], + snd_rounded + 8, i); + if (srdis_mask ^ srdis_multi) { + break; + } + srdis_mask = (srdis_mask << i) | srdis_mask; + rsize++; + } + } + if (srdis) { + continue; + } + if (rsize < TARGET_PAGE_BITS) { + *page_size = 1 << rsize; + } + break; + } + + if (n == -1) { /* no hits */ + if (!pmsav7_use_background_region(cpu, mmu_idx, is_user)) { + /* background fault */ + fi->type = ARMFault_Background; + return true; + } + get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); + } else { /* a MPU hit! */ + uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3); + uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1); + + if (m_is_system_region(env, address)) { + /* System space is always execute never */ + xn = 1; + } + + if (is_user) { /* User mode AP bit decoding */ + switch (ap) { + case 0: + case 1: + case 5: + break; /* no access */ + case 3: + *prot |= PAGE_WRITE; + /* fall through */ + case 2: + case 6: + *prot |= PAGE_READ | PAGE_EXEC; + break; + case 7: + /* for v7M, same as 6; for R profile a reserved value */ + if (arm_feature(env, ARM_FEATURE_M)) { + *prot |= PAGE_READ | PAGE_EXEC; + break; + } + /* fall through */ + default: + qemu_log_mask(LOG_GUEST_ERROR, + "DRACR[%d]: Bad value for AP bits: 0x%" + PRIx32 "\n", n, ap); + } + } else { /* Priv. mode AP bits decoding */ + switch (ap) { + case 0: + break; /* no access */ + case 1: + case 2: + case 3: + *prot |= PAGE_WRITE; + /* fall through */ + case 5: + case 6: + *prot |= PAGE_READ | PAGE_EXEC; + break; + case 7: + /* for v7M, same as 6; for R profile a reserved value */ + if (arm_feature(env, ARM_FEATURE_M)) { + *prot |= PAGE_READ | PAGE_EXEC; + break; + } + /* fall through */ + default: + qemu_log_mask(LOG_GUEST_ERROR, + "DRACR[%d]: Bad value for AP bits: 0x%" + PRIx32 "\n", n, ap); + } + } + + /* execute never */ + if (xn) { + *prot &= ~PAGE_EXEC; + } + } + } + + fi->type = ARMFault_Permission; + fi->level = 1; + return !(*prot & (1 << access_type)); +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d6e3fee152..d24b7c263a 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,14 +33,14 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } +bool m_is_ppb_region(CPUARMState *env, uint32_t address); +bool m_is_system_region(CPUARMState *env, uint32_t address); + void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, int32_t address, int *prot); -bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi); +bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); + bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *txattrs, From 730d5c31d8872e94d08877ad62f44cc49bc01c90 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 15/55] target/arm: Move get_phys_addr_pmsav8 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-9-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 75 ------------------------------------------- target/arm/ptw.c | 77 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 5 --- 3 files changed, 77 insertions(+), 80 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 2ebaf69407..44997fd179 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11970,81 +11970,6 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } - -bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, target_ulong *page_size, - ARMMMUFaultInfo *fi) -{ - uint32_t secure = regime_is_secure(env, mmu_idx); - V8M_SAttributes sattrs = {}; - bool ret; - bool mpu_is_subpage; - - if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { - v8m_security_lookup(env, address, access_type, mmu_idx, &sattrs); - if (access_type == MMU_INST_FETCH) { - /* Instruction fetches always use the MMU bank and the - * transaction attribute determined by the fetch address, - * regardless of CPU state. This is painful for QEMU - * to handle, because it would mean we need to encode - * into the mmu_idx not just the (user, negpri) information - * for the current security state but also that for the - * other security state, which would balloon the number - * of mmu_idx values needed alarmingly. - * Fortunately we can avoid this because it's not actually - * possible to arbitrarily execute code from memory with - * the wrong security attribute: it will always generate - * an exception of some kind or another, apart from the - * special case of an NS CPU executing an SG instruction - * in S&NSC memory. So we always just fail the translation - * here and sort things out in the exception handler - * (including possibly emulating an SG instruction). - */ - if (sattrs.ns != !secure) { - if (sattrs.nsc) { - fi->type = ARMFault_QEMU_NSCExec; - } else { - fi->type = ARMFault_QEMU_SFault; - } - *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; - *phys_ptr = address; - *prot = 0; - return true; - } - } else { - /* For data accesses we always use the MMU bank indicated - * by the current CPU state, but the security attributes - * might downgrade a secure access to nonsecure. - */ - if (sattrs.ns) { - txattrs->secure = false; - } else if (!secure) { - /* NS access to S memory must fault. - * Architecturally we should first check whether the - * MPU information for this address indicates that we - * are doing an unaligned access to Device memory, which - * should generate a UsageFault instead. QEMU does not - * currently check for that kind of unaligned access though. - * If we added it we would need to do so as a special case - * for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt(). - */ - fi->type = ARMFault_QEMU_SFault; - *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; - *phys_ptr = address; - *prot = 0; - return true; - } - } - } - - ret = pmsav8_mpu_lookup(env, address, access_type, mmu_idx, phys_ptr, - txattrs, prot, &mpu_is_subpage, fi, NULL); - *page_size = sattrs.subpage || mpu_is_subpage ? 1 : TARGET_PAGE_SIZE; - return ret; -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 27715dbfa8..28caa7a7ae 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -605,6 +605,83 @@ static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } +static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, target_ulong *page_size, + ARMMMUFaultInfo *fi) +{ + uint32_t secure = regime_is_secure(env, mmu_idx); + V8M_SAttributes sattrs = {}; + bool ret; + bool mpu_is_subpage; + + if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { + v8m_security_lookup(env, address, access_type, mmu_idx, &sattrs); + if (access_type == MMU_INST_FETCH) { + /* + * Instruction fetches always use the MMU bank and the + * transaction attribute determined by the fetch address, + * regardless of CPU state. This is painful for QEMU + * to handle, because it would mean we need to encode + * into the mmu_idx not just the (user, negpri) information + * for the current security state but also that for the + * other security state, which would balloon the number + * of mmu_idx values needed alarmingly. + * Fortunately we can avoid this because it's not actually + * possible to arbitrarily execute code from memory with + * the wrong security attribute: it will always generate + * an exception of some kind or another, apart from the + * special case of an NS CPU executing an SG instruction + * in S&NSC memory. So we always just fail the translation + * here and sort things out in the exception handler + * (including possibly emulating an SG instruction). + */ + if (sattrs.ns != !secure) { + if (sattrs.nsc) { + fi->type = ARMFault_QEMU_NSCExec; + } else { + fi->type = ARMFault_QEMU_SFault; + } + *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; + *phys_ptr = address; + *prot = 0; + return true; + } + } else { + /* + * For data accesses we always use the MMU bank indicated + * by the current CPU state, but the security attributes + * might downgrade a secure access to nonsecure. + */ + if (sattrs.ns) { + txattrs->secure = false; + } else if (!secure) { + /* + * NS access to S memory must fault. + * Architecturally we should first check whether the + * MPU information for this address indicates that we + * are doing an unaligned access to Device memory, which + * should generate a UsageFault instead. QEMU does not + * currently check for that kind of unaligned access though. + * If we added it we would need to do so as a special case + * for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt(). + */ + fi->type = ARMFault_QEMU_SFault; + *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; + *phys_ptr = address; + *prot = 0; + return true; + } + } + } + + ret = pmsav8_mpu_lookup(env, address, access_type, mmu_idx, phys_ptr, + txattrs, prot, &mpu_is_subpage, fi, NULL); + *page_size = sattrs.subpage || mpu_is_subpage ? 1 : TARGET_PAGE_SIZE; + return ret; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d24b7c263a..d569507951 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -41,11 +41,6 @@ void get_phys_addr_pmsav7_default(CPUARMState *env, int32_t address, int *prot); bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); -bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, target_ulong *page_size, - ARMMMUFaultInfo *fi); bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From fedbaa0503d2c56d4ef58ca9a08121e2af7f5e54 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 16/55] target/arm: Move pmsav8_mpu_lookup to ptw.c This is the final user of get_phys_addr_pmsav7_default within helper.c, so make it static within ptw.c. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-10-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 136 ----------------------------------------- target/arm/ptw.c | 146 +++++++++++++++++++++++++++++++++++++++++++- target/arm/ptw.h | 3 - 3 files changed, 143 insertions(+), 142 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 44997fd179..cb23413d8e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11834,142 +11834,6 @@ void v8m_security_lookup(CPUARMState *env, uint32_t address, } } -bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, bool *is_subpage, - ARMMMUFaultInfo *fi, uint32_t *mregion) -{ - /* Perform a PMSAv8 MPU lookup (without also doing the SAU check - * that a full phys-to-virt translation does). - * mregion is (if not NULL) set to the region number which matched, - * or -1 if no region number is returned (MPU off, address did not - * hit a region, address hit in multiple regions). - * We set is_subpage to true if the region hit doesn't cover the - * entire TARGET_PAGE the address is within. - */ - ARMCPU *cpu = env_archcpu(env); - bool is_user = regime_is_user(env, mmu_idx); - uint32_t secure = regime_is_secure(env, mmu_idx); - int n; - int matchregion = -1; - bool hit = false; - uint32_t addr_page_base = address & TARGET_PAGE_MASK; - uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); - - *is_subpage = false; - *phys_ptr = address; - *prot = 0; - if (mregion) { - *mregion = -1; - } - - /* Unlike the ARM ARM pseudocode, we don't need to check whether this - * was an exception vector read from the vector table (which is always - * done using the default system address map), because those accesses - * are done in arm_v7m_load_vector(), which always does a direct - * read using address_space_ldl(), rather than going via this function. - */ - if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */ - hit = true; - } else if (m_is_ppb_region(env, address)) { - hit = true; - } else { - if (pmsav7_use_background_region(cpu, mmu_idx, is_user)) { - hit = true; - } - - for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { - /* region search */ - /* Note that the base address is bits [31:5] from the register - * with bits [4:0] all zeroes, but the limit address is bits - * [31:5] from the register with bits [4:0] all ones. - */ - uint32_t base = env->pmsav8.rbar[secure][n] & ~0x1f; - uint32_t limit = env->pmsav8.rlar[secure][n] | 0x1f; - - if (!(env->pmsav8.rlar[secure][n] & 0x1)) { - /* Region disabled */ - continue; - } - - if (address < base || address > limit) { - /* - * Address not in this region. We must check whether the - * region covers addresses in the same page as our address. - * In that case we must not report a size that covers the - * whole page for a subsequent hit against a different MPU - * region or the background region, because it would result in - * incorrect TLB hits for subsequent accesses to addresses that - * are in this MPU region. - */ - if (limit >= base && - ranges_overlap(base, limit - base + 1, - addr_page_base, - TARGET_PAGE_SIZE)) { - *is_subpage = true; - } - continue; - } - - if (base > addr_page_base || limit < addr_page_limit) { - *is_subpage = true; - } - - if (matchregion != -1) { - /* Multiple regions match -- always a failure (unlike - * PMSAv7 where highest-numbered-region wins) - */ - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - - matchregion = n; - hit = true; - } - } - - if (!hit) { - /* background fault */ - fi->type = ARMFault_Background; - return true; - } - - if (matchregion == -1) { - /* hit using the background region */ - get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); - } else { - uint32_t ap = extract32(env->pmsav8.rbar[secure][matchregion], 1, 2); - uint32_t xn = extract32(env->pmsav8.rbar[secure][matchregion], 0, 1); - bool pxn = false; - - if (arm_feature(env, ARM_FEATURE_V8_1M)) { - pxn = extract32(env->pmsav8.rlar[secure][matchregion], 4, 1); - } - - if (m_is_system_region(env, address)) { - /* System space is always execute never */ - xn = 1; - } - - *prot = simple_ap_to_rw_prot(env, mmu_idx, ap); - if (*prot && !xn && !(pxn && !is_user)) { - *prot |= PAGE_EXEC; - } - /* We don't need to look the attribute up in the MAIR0/MAIR1 - * registers because that only tells us about cacheability. - */ - if (mregion) { - *mregion = matchregion; - } - } - - fi->type = ARMFault_Permission; - fi->level = 1; - return !(*prot & (1 << access_type)); -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 28caa7a7ae..989e783cce 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -375,9 +375,8 @@ static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, return false; } -void get_phys_addr_pmsav7_default(CPUARMState *env, - ARMMMUIdx mmu_idx, - int32_t address, int *prot) +static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, + int32_t address, int *prot) { if (!arm_feature(env, ARM_FEATURE_M)) { *prot = PAGE_READ | PAGE_WRITE; @@ -605,6 +604,147 @@ static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } +bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, bool *is_subpage, + ARMMMUFaultInfo *fi, uint32_t *mregion) +{ + /* + * Perform a PMSAv8 MPU lookup (without also doing the SAU check + * that a full phys-to-virt translation does). + * mregion is (if not NULL) set to the region number which matched, + * or -1 if no region number is returned (MPU off, address did not + * hit a region, address hit in multiple regions). + * We set is_subpage to true if the region hit doesn't cover the + * entire TARGET_PAGE the address is within. + */ + ARMCPU *cpu = env_archcpu(env); + bool is_user = regime_is_user(env, mmu_idx); + uint32_t secure = regime_is_secure(env, mmu_idx); + int n; + int matchregion = -1; + bool hit = false; + uint32_t addr_page_base = address & TARGET_PAGE_MASK; + uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); + + *is_subpage = false; + *phys_ptr = address; + *prot = 0; + if (mregion) { + *mregion = -1; + } + + /* + * Unlike the ARM ARM pseudocode, we don't need to check whether this + * was an exception vector read from the vector table (which is always + * done using the default system address map), because those accesses + * are done in arm_v7m_load_vector(), which always does a direct + * read using address_space_ldl(), rather than going via this function. + */ + if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */ + hit = true; + } else if (m_is_ppb_region(env, address)) { + hit = true; + } else { + if (pmsav7_use_background_region(cpu, mmu_idx, is_user)) { + hit = true; + } + + for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { + /* region search */ + /* + * Note that the base address is bits [31:5] from the register + * with bits [4:0] all zeroes, but the limit address is bits + * [31:5] from the register with bits [4:0] all ones. + */ + uint32_t base = env->pmsav8.rbar[secure][n] & ~0x1f; + uint32_t limit = env->pmsav8.rlar[secure][n] | 0x1f; + + if (!(env->pmsav8.rlar[secure][n] & 0x1)) { + /* Region disabled */ + continue; + } + + if (address < base || address > limit) { + /* + * Address not in this region. We must check whether the + * region covers addresses in the same page as our address. + * In that case we must not report a size that covers the + * whole page for a subsequent hit against a different MPU + * region or the background region, because it would result in + * incorrect TLB hits for subsequent accesses to addresses that + * are in this MPU region. + */ + if (limit >= base && + ranges_overlap(base, limit - base + 1, + addr_page_base, + TARGET_PAGE_SIZE)) { + *is_subpage = true; + } + continue; + } + + if (base > addr_page_base || limit < addr_page_limit) { + *is_subpage = true; + } + + if (matchregion != -1) { + /* + * Multiple regions match -- always a failure (unlike + * PMSAv7 where highest-numbered-region wins) + */ + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + + matchregion = n; + hit = true; + } + } + + if (!hit) { + /* background fault */ + fi->type = ARMFault_Background; + return true; + } + + if (matchregion == -1) { + /* hit using the background region */ + get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); + } else { + uint32_t ap = extract32(env->pmsav8.rbar[secure][matchregion], 1, 2); + uint32_t xn = extract32(env->pmsav8.rbar[secure][matchregion], 0, 1); + bool pxn = false; + + if (arm_feature(env, ARM_FEATURE_V8_1M)) { + pxn = extract32(env->pmsav8.rlar[secure][matchregion], 4, 1); + } + + if (m_is_system_region(env, address)) { + /* System space is always execute never */ + xn = 1; + } + + *prot = simple_ap_to_rw_prot(env, mmu_idx, ap); + if (*prot && !xn && !(pxn && !is_user)) { + *prot |= PAGE_EXEC; + } + /* + * We don't need to look the attribute up in the MAIR0/MAIR1 + * registers because that only tells us about cacheability. + */ + if (mregion) { + *mregion = matchregion; + } + } + + fi->type = ARMFault_Permission; + fi->level = 1; + return !(*prot & (1 << access_type)); +} + static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *txattrs, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d569507951..8d2e239714 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -36,9 +36,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) bool m_is_ppb_region(CPUARMState *env, uint32_t address); bool m_is_system_region(CPUARMState *env, uint32_t address); -void get_phys_addr_pmsav7_default(CPUARMState *env, - ARMMMUIdx mmu_idx, - int32_t address, int *prot); bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, From c8e436c9f72bca95e37a155d6ead42d55f74eed2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 17/55] target/arm: Move pmsav7_use_background_region to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-11-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 19 ------------------- target/arm/ptw.c | 21 +++++++++++++++++++++ target/arm/ptw.h | 2 -- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index cb23413d8e..62e48f0925 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11678,25 +11678,6 @@ do_fault: return true; } -bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) -{ - /* Return true if we should use the default memory map as a - * "background" region if there are no hits against any MPU regions. - */ - CPUARMState *env = &cpu->env; - - if (is_user) { - return false; - } - - if (arm_feature(env, ARM_FEATURE_M)) { - return env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] - & R_V7M_MPU_CTRL_PRIVDEFENA_MASK; - } else { - return regime_sctlr(env, mmu_idx) & SCTLR_BR; - } -} - bool m_is_ppb_region(CPUARMState *env, uint32_t address) { /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 989e783cce..b82638b5a0 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -415,6 +415,27 @@ static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, } } +static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, + bool is_user) +{ + /* + * Return true if we should use the default memory map as a + * "background" region if there are no hits against any MPU regions. + */ + CPUARMState *env = &cpu->env; + + if (is_user) { + return false; + } + + if (arm_feature(env, ARM_FEATURE_M)) { + return env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] + & R_V7M_MPU_CTRL_PRIVDEFENA_MASK; + } else { + return regime_sctlr(env, mmu_idx) & SCTLR_BR; + } +} + static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 8d2e239714..d2d2711908 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -36,8 +36,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) bool m_is_ppb_region(CPUARMState *env, uint32_t address); bool m_is_system_region(CPUARMState *env, uint32_t address); -bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); - bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From 2c1f429df38822ce6955224d784f1567f8af219f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 18/55] target/arm: Move v8m_security_lookup to ptw.c This function has one private helper, v8m_is_sau_exempt, so move that at the same time. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-12-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 123 ------------------------------------------ target/arm/ptw.c | 126 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 123 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 62e48f0925..d6a749ad0e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -9,7 +9,6 @@ #include "qemu/osdep.h" #include "qemu/units.h" #include "qemu/log.h" -#include "target/arm/idau.h" #include "trace.h" #include "cpu.h" #include "internals.h" @@ -11693,128 +11692,6 @@ bool m_is_system_region(CPUARMState *env, uint32_t address) return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; } -static bool v8m_is_sau_exempt(CPUARMState *env, - uint32_t address, MMUAccessType access_type) -{ - /* The architecture specifies that certain address ranges are - * exempt from v8M SAU/IDAU checks. - */ - return - (access_type == MMU_INST_FETCH && m_is_system_region(env, address)) || - (address >= 0xe0000000 && address <= 0xe0002fff) || - (address >= 0xe000e000 && address <= 0xe000efff) || - (address >= 0xe002e000 && address <= 0xe002efff) || - (address >= 0xe0040000 && address <= 0xe0041fff) || - (address >= 0xe00ff000 && address <= 0xe00fffff); -} - -void v8m_security_lookup(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - V8M_SAttributes *sattrs) -{ - /* Look up the security attributes for this address. Compare the - * pseudocode SecurityCheck() function. - * We assume the caller has zero-initialized *sattrs. - */ - ARMCPU *cpu = env_archcpu(env); - int r; - bool idau_exempt = false, idau_ns = true, idau_nsc = true; - int idau_region = IREGION_NOTVALID; - uint32_t addr_page_base = address & TARGET_PAGE_MASK; - uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); - - if (cpu->idau) { - IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau); - IDAUInterface *ii = IDAU_INTERFACE(cpu->idau); - - iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns, - &idau_nsc); - } - - if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) { - /* 0xf0000000..0xffffffff is always S for insn fetches */ - return; - } - - if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) { - sattrs->ns = !regime_is_secure(env, mmu_idx); - return; - } - - if (idau_region != IREGION_NOTVALID) { - sattrs->irvalid = true; - sattrs->iregion = idau_region; - } - - switch (env->sau.ctrl & 3) { - case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */ - break; - case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */ - sattrs->ns = true; - break; - default: /* SAU.ENABLE == 1 */ - for (r = 0; r < cpu->sau_sregion; r++) { - if (env->sau.rlar[r] & 1) { - uint32_t base = env->sau.rbar[r] & ~0x1f; - uint32_t limit = env->sau.rlar[r] | 0x1f; - - if (base <= address && limit >= address) { - if (base > addr_page_base || limit < addr_page_limit) { - sattrs->subpage = true; - } - if (sattrs->srvalid) { - /* If we hit in more than one region then we must report - * as Secure, not NS-Callable, with no valid region - * number info. - */ - sattrs->ns = false; - sattrs->nsc = false; - sattrs->sregion = 0; - sattrs->srvalid = false; - break; - } else { - if (env->sau.rlar[r] & 2) { - sattrs->nsc = true; - } else { - sattrs->ns = true; - } - sattrs->srvalid = true; - sattrs->sregion = r; - } - } else { - /* - * Address not in this region. We must check whether the - * region covers addresses in the same page as our address. - * In that case we must not report a size that covers the - * whole page for a subsequent hit against a different MPU - * region or the background region, because it would result - * in incorrect TLB hits for subsequent accesses to - * addresses that are in this MPU region. - */ - if (limit >= base && - ranges_overlap(base, limit - base + 1, - addr_page_base, - TARGET_PAGE_SIZE)) { - sattrs->subpage = true; - } - } - } - } - break; - } - - /* - * The IDAU will override the SAU lookup results if it specifies - * higher security than the SAU does. - */ - if (!idau_ns) { - if (sattrs->ns || (!idau_nsc && sattrs->nsc)) { - sattrs->ns = false; - sattrs->nsc = idau_nsc; - } - } -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index b82638b5a0..c15fba43c3 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -11,6 +11,7 @@ #include "qemu/range.h" #include "cpu.h" #include "internals.h" +#include "idau.h" #include "ptw.h" @@ -766,6 +767,131 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } +static bool v8m_is_sau_exempt(CPUARMState *env, + uint32_t address, MMUAccessType access_type) +{ + /* + * The architecture specifies that certain address ranges are + * exempt from v8M SAU/IDAU checks. + */ + return + (access_type == MMU_INST_FETCH && m_is_system_region(env, address)) || + (address >= 0xe0000000 && address <= 0xe0002fff) || + (address >= 0xe000e000 && address <= 0xe000efff) || + (address >= 0xe002e000 && address <= 0xe002efff) || + (address >= 0xe0040000 && address <= 0xe0041fff) || + (address >= 0xe00ff000 && address <= 0xe00fffff); +} + +void v8m_security_lookup(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + V8M_SAttributes *sattrs) +{ + /* + * Look up the security attributes for this address. Compare the + * pseudocode SecurityCheck() function. + * We assume the caller has zero-initialized *sattrs. + */ + ARMCPU *cpu = env_archcpu(env); + int r; + bool idau_exempt = false, idau_ns = true, idau_nsc = true; + int idau_region = IREGION_NOTVALID; + uint32_t addr_page_base = address & TARGET_PAGE_MASK; + uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); + + if (cpu->idau) { + IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau); + IDAUInterface *ii = IDAU_INTERFACE(cpu->idau); + + iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns, + &idau_nsc); + } + + if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) { + /* 0xf0000000..0xffffffff is always S for insn fetches */ + return; + } + + if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) { + sattrs->ns = !regime_is_secure(env, mmu_idx); + return; + } + + if (idau_region != IREGION_NOTVALID) { + sattrs->irvalid = true; + sattrs->iregion = idau_region; + } + + switch (env->sau.ctrl & 3) { + case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */ + break; + case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */ + sattrs->ns = true; + break; + default: /* SAU.ENABLE == 1 */ + for (r = 0; r < cpu->sau_sregion; r++) { + if (env->sau.rlar[r] & 1) { + uint32_t base = env->sau.rbar[r] & ~0x1f; + uint32_t limit = env->sau.rlar[r] | 0x1f; + + if (base <= address && limit >= address) { + if (base > addr_page_base || limit < addr_page_limit) { + sattrs->subpage = true; + } + if (sattrs->srvalid) { + /* + * If we hit in more than one region then we must report + * as Secure, not NS-Callable, with no valid region + * number info. + */ + sattrs->ns = false; + sattrs->nsc = false; + sattrs->sregion = 0; + sattrs->srvalid = false; + break; + } else { + if (env->sau.rlar[r] & 2) { + sattrs->nsc = true; + } else { + sattrs->ns = true; + } + sattrs->srvalid = true; + sattrs->sregion = r; + } + } else { + /* + * Address not in this region. We must check whether the + * region covers addresses in the same page as our address. + * In that case we must not report a size that covers the + * whole page for a subsequent hit against a different MPU + * region or the background region, because it would result + * in incorrect TLB hits for subsequent accesses to + * addresses that are in this MPU region. + */ + if (limit >= base && + ranges_overlap(base, limit - base + 1, + addr_page_base, + TARGET_PAGE_SIZE)) { + sattrs->subpage = true; + } + } + } + } + break; + } + + /* + * The IDAU will override the SAU lookup results if it specifies + * higher security than the SAU does. + */ + if (!idau_ns) { + if (sattrs->ns || (!idau_nsc && sattrs->nsc)) { + sattrs->ns = false; + sattrs->nsc = idau_nsc; + } + } +} + static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *txattrs, From 47ff5ba9d08c2c769794a59b88f1c19ada004730 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 19/55] target/arm: Move m_is_{ppb,system}_region to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-13-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 15 --------------- target/arm/ptw.c | 16 ++++++++++++++++ target/arm/ptw.h | 3 --- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d6a749ad0e..d2ef12346b 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11677,21 +11677,6 @@ do_fault: return true; } -bool m_is_ppb_region(CPUARMState *env, uint32_t address) -{ - /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ - return arm_feature(env, ARM_FEATURE_M) && - extract32(address, 20, 12) == 0xe00; -} - -bool m_is_system_region(CPUARMState *env, uint32_t address) -{ - /* True if address is in the M profile system region - * 0xe0000000 - 0xffffffff - */ - return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index c15fba43c3..32ba2e5e8b 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -416,6 +416,22 @@ static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, } } +static bool m_is_ppb_region(CPUARMState *env, uint32_t address) +{ + /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ + return arm_feature(env, ARM_FEATURE_M) && + extract32(address, 20, 12) == 0xe00; +} + +static bool m_is_system_region(CPUARMState *env, uint32_t address) +{ + /* + * True if address is in the M profile system region + * 0xe0000000 - 0xffffffff + */ + return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; +} + static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) { diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d2d2711908..6c47a57599 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,9 +33,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } -bool m_is_ppb_region(CPUARMState *env, uint32_t address); -bool m_is_system_region(CPUARMState *env, uint32_t address); - bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From 4c74ab157b056710b043a02c8101c449c179ae11 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 20/55] target/arm: Move get_level1_table_address to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-14-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 26 +------------------------- target/arm/ptw.c | 23 +++++++++++++++++++++++ target/arm/ptw.h | 4 ++-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d2ef12346b..a144cb2641 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10482,8 +10482,7 @@ static inline bool regime_translation_big_endian(CPUARMState *env, } /* Return the TTBR associated with this translation regime */ -static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, - int ttbrn) +uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { if (mmu_idx == ARMMMUIdx_Stage2) { return env->cp15.vttbr_el2; @@ -10774,29 +10773,6 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } -bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, - uint32_t *table, uint32_t address) -{ - /* Note that we can only get here for an AArch32 PL0/PL1 lookup */ - TCR *tcr = regime_tcr(env, mmu_idx); - - if (address & tcr->mask) { - if (tcr->raw_tcr & TTBCR_PD1) { - /* Translation table walk disabled for TTBR1 */ - return false; - } - *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000; - } else { - if (tcr->raw_tcr & TTBCR_PD0) { - /* Translation table walk disabled for TTBR0 */ - return false; - } - *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask; - } - *table |= (address >> 18) & 0x3ffc; - return true; -} - static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 32ba2e5e8b..5737a3976b 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -15,6 +15,29 @@ #include "ptw.h" +static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, + uint32_t *table, uint32_t address) +{ + /* Note that we can only get here for an AArch32 PL0/PL1 lookup */ + TCR *tcr = regime_tcr(env, mmu_idx); + + if (address & tcr->mask) { + if (tcr->raw_tcr & TTBCR_PD1) { + /* Translation table walk disabled for TTBR1 */ + return false; + } + *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000; + } else { + if (tcr->raw_tcr & TTBCR_PD0) { + /* Translation table walk disabled for TTBR0 */ + return false; + } + *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask; + } + *table |= (address >> 18) & 0x3ffc; + return true; +} + static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 6c47a57599..dd6fb93f33 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -18,11 +18,11 @@ uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); +uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); + ARMCacheAttrs combine_cacheattrs(CPUARMState *env, ARMCacheAttrs s1, ARMCacheAttrs s2); -bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, - uint32_t *table, uint32_t address); int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot); int simple_ap_to_rw_prot_is_user(int ap, bool is_user); From 966f4bb7d88554ed74e1006c516ddeef37e523d6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 21/55] target/arm: Move combine_cacheattrs and subroutines to ptw.c There are a handful of helpers for combine_cacheattrs that we can move at the same time as the main entry point. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-15-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 218 ------------------------------------------- target/arm/ptw.c | 221 ++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 - 3 files changed, 221 insertions(+), 221 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index a144cb2641..dab485e64a 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10977,36 +10977,6 @@ static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, } return true; } - -/* Translate from the 4-bit stage 2 representation of - * memory attributes (without cache-allocation hints) to - * the 8-bit representation of the stage 1 MAIR registers - * (which includes allocation hints). - * - * ref: shared/translation/attrs/S2AttrDecode() - * .../S2ConvertAttrsHints() - */ -static uint8_t convert_stage2_attrs(CPUARMState *env, uint8_t s2attrs) -{ - uint8_t hiattr = extract32(s2attrs, 2, 2); - uint8_t loattr = extract32(s2attrs, 0, 2); - uint8_t hihint = 0, lohint = 0; - - if (hiattr != 0) { /* normal memory */ - if (arm_hcr_el2_eff(env) & HCR_CD) { /* cache disabled */ - hiattr = loattr = 1; /* non-cacheable */ - } else { - if (hiattr != 1) { /* Write-through or write-back */ - hihint = 3; /* RW allocate */ - } - if (loattr != 1) { /* Write-through or write-back */ - lohint = 3; /* RW allocate */ - } - } - } - - return (hiattr << 6) | (hihint << 4) | (loattr << 2) | lohint; -} #endif /* !CONFIG_USER_ONLY */ /* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ @@ -11653,194 +11623,6 @@ do_fault: return true; } -/* Combine either inner or outer cacheability attributes for normal - * memory, according to table D4-42 and pseudocode procedure - * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). - * - * NB: only stage 1 includes allocation hints (RW bits), leading to - * some asymmetry. - */ -static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2) -{ - if (s1 == 4 || s2 == 4) { - /* non-cacheable has precedence */ - return 4; - } else if (extract32(s1, 2, 2) == 0 || extract32(s1, 2, 2) == 2) { - /* stage 1 write-through takes precedence */ - return s1; - } else if (extract32(s2, 2, 2) == 2) { - /* stage 2 write-through takes precedence, but the allocation hint - * is still taken from stage 1 - */ - return (2 << 2) | extract32(s1, 0, 2); - } else { /* write-back */ - return s1; - } -} - -/* - * Combine the memory type and cacheability attributes of - * s1 and s2 for the HCR_EL2.FWB == 0 case, returning the - * combined attributes in MAIR_EL1 format. - */ -static uint8_t combined_attrs_nofwb(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) -{ - uint8_t s1lo, s2lo, s1hi, s2hi, s2_mair_attrs, ret_attrs; - - s2_mair_attrs = convert_stage2_attrs(env, s2.attrs); - - s1lo = extract32(s1.attrs, 0, 4); - s2lo = extract32(s2_mair_attrs, 0, 4); - s1hi = extract32(s1.attrs, 4, 4); - s2hi = extract32(s2_mair_attrs, 4, 4); - - /* Combine memory type and cacheability attributes */ - if (s1hi == 0 || s2hi == 0) { - /* Device has precedence over normal */ - if (s1lo == 0 || s2lo == 0) { - /* nGnRnE has precedence over anything */ - ret_attrs = 0; - } else if (s1lo == 4 || s2lo == 4) { - /* non-Reordering has precedence over Reordering */ - ret_attrs = 4; /* nGnRE */ - } else if (s1lo == 8 || s2lo == 8) { - /* non-Gathering has precedence over Gathering */ - ret_attrs = 8; /* nGRE */ - } else { - ret_attrs = 0xc; /* GRE */ - } - } else { /* Normal memory */ - /* Outer/inner cacheability combine independently */ - ret_attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4 - | combine_cacheattr_nibble(s1lo, s2lo); - } - return ret_attrs; -} - -static uint8_t force_cacheattr_nibble_wb(uint8_t attr) -{ - /* - * Given the 4 bits specifying the outer or inner cacheability - * in MAIR format, return a value specifying Normal Write-Back, - * with the allocation and transient hints taken from the input - * if the input specified some kind of cacheable attribute. - */ - if (attr == 0 || attr == 4) { - /* - * 0 == an UNPREDICTABLE encoding - * 4 == Non-cacheable - * Either way, force Write-Back RW allocate non-transient - */ - return 0xf; - } - /* Change WriteThrough to WriteBack, keep allocation and transient hints */ - return attr | 4; -} - -/* - * Combine the memory type and cacheability attributes of - * s1 and s2 for the HCR_EL2.FWB == 1 case, returning the - * combined attributes in MAIR_EL1 format. - */ -static uint8_t combined_attrs_fwb(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) -{ - switch (s2.attrs) { - case 7: - /* Use stage 1 attributes */ - return s1.attrs; - case 6: - /* - * Force Normal Write-Back. Note that if S1 is Normal cacheable - * then we take the allocation hints from it; otherwise it is - * RW allocate, non-transient. - */ - if ((s1.attrs & 0xf0) == 0) { - /* S1 is Device */ - return 0xff; - } - /* Need to check the Inner and Outer nibbles separately */ - return force_cacheattr_nibble_wb(s1.attrs & 0xf) | - force_cacheattr_nibble_wb(s1.attrs >> 4) << 4; - case 5: - /* If S1 attrs are Device, use them; otherwise Normal Non-cacheable */ - if ((s1.attrs & 0xf0) == 0) { - return s1.attrs; - } - return 0x44; - case 0 ... 3: - /* Force Device, of subtype specified by S2 */ - return s2.attrs << 2; - default: - /* - * RESERVED values (including RES0 descriptor bit [5] being nonzero); - * arbitrarily force Device. - */ - return 0; - } -} - -/* Combine S1 and S2 cacheability/shareability attributes, per D4.5.4 - * and CombineS1S2Desc() - * - * @env: CPUARMState - * @s1: Attributes from stage 1 walk - * @s2: Attributes from stage 2 walk - */ -ARMCacheAttrs combine_cacheattrs(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) -{ - ARMCacheAttrs ret; - bool tagged = false; - - assert(s2.is_s2_format && !s1.is_s2_format); - ret.is_s2_format = false; - - if (s1.attrs == 0xf0) { - tagged = true; - s1.attrs = 0xff; - } - - /* Combine shareability attributes (table D4-43) */ - if (s1.shareability == 2 || s2.shareability == 2) { - /* if either are outer-shareable, the result is outer-shareable */ - ret.shareability = 2; - } else if (s1.shareability == 3 || s2.shareability == 3) { - /* if either are inner-shareable, the result is inner-shareable */ - ret.shareability = 3; - } else { - /* both non-shareable */ - ret.shareability = 0; - } - - /* Combine memory type and cacheability attributes */ - if (arm_hcr_el2_eff(env) & HCR_FWB) { - ret.attrs = combined_attrs_fwb(env, s1, s2); - } else { - ret.attrs = combined_attrs_nofwb(env, s1, s2); - } - - /* - * Any location for which the resultant memory type is any - * type of Device memory is always treated as Outer Shareable. - * Any location for which the resultant memory type is Normal - * Inner Non-cacheable, Outer Non-cacheable is always treated - * as Outer Shareable. - * TODO: FEAT_XS adds another value (0x40) also meaning iNCoNC - */ - if ((ret.attrs & 0xf0) == 0 || ret.attrs == 0x44) { - ret.shareability = 2; - } - - /* TODO: CombineS1S2Desc does not consider transient, only WB, RWA. */ - if (tagged && ret.attrs == 0xff) { - ret.attrs = 0xf0; - } - - return ret; -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 5737a3976b..f2ca2bb8fe 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -1008,6 +1008,227 @@ static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, return ret; } +/* + * Translate from the 4-bit stage 2 representation of + * memory attributes (without cache-allocation hints) to + * the 8-bit representation of the stage 1 MAIR registers + * (which includes allocation hints). + * + * ref: shared/translation/attrs/S2AttrDecode() + * .../S2ConvertAttrsHints() + */ +static uint8_t convert_stage2_attrs(CPUARMState *env, uint8_t s2attrs) +{ + uint8_t hiattr = extract32(s2attrs, 2, 2); + uint8_t loattr = extract32(s2attrs, 0, 2); + uint8_t hihint = 0, lohint = 0; + + if (hiattr != 0) { /* normal memory */ + if (arm_hcr_el2_eff(env) & HCR_CD) { /* cache disabled */ + hiattr = loattr = 1; /* non-cacheable */ + } else { + if (hiattr != 1) { /* Write-through or write-back */ + hihint = 3; /* RW allocate */ + } + if (loattr != 1) { /* Write-through or write-back */ + lohint = 3; /* RW allocate */ + } + } + } + + return (hiattr << 6) | (hihint << 4) | (loattr << 2) | lohint; +} + +/* + * Combine either inner or outer cacheability attributes for normal + * memory, according to table D4-42 and pseudocode procedure + * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). + * + * NB: only stage 1 includes allocation hints (RW bits), leading to + * some asymmetry. + */ +static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2) +{ + if (s1 == 4 || s2 == 4) { + /* non-cacheable has precedence */ + return 4; + } else if (extract32(s1, 2, 2) == 0 || extract32(s1, 2, 2) == 2) { + /* stage 1 write-through takes precedence */ + return s1; + } else if (extract32(s2, 2, 2) == 2) { + /* stage 2 write-through takes precedence, but the allocation hint + * is still taken from stage 1 + */ + return (2 << 2) | extract32(s1, 0, 2); + } else { /* write-back */ + return s1; + } +} + +/* + * Combine the memory type and cacheability attributes of + * s1 and s2 for the HCR_EL2.FWB == 0 case, returning the + * combined attributes in MAIR_EL1 format. + */ +static uint8_t combined_attrs_nofwb(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) +{ + uint8_t s1lo, s2lo, s1hi, s2hi, s2_mair_attrs, ret_attrs; + + s2_mair_attrs = convert_stage2_attrs(env, s2.attrs); + + s1lo = extract32(s1.attrs, 0, 4); + s2lo = extract32(s2_mair_attrs, 0, 4); + s1hi = extract32(s1.attrs, 4, 4); + s2hi = extract32(s2_mair_attrs, 4, 4); + + /* Combine memory type and cacheability attributes */ + if (s1hi == 0 || s2hi == 0) { + /* Device has precedence over normal */ + if (s1lo == 0 || s2lo == 0) { + /* nGnRnE has precedence over anything */ + ret_attrs = 0; + } else if (s1lo == 4 || s2lo == 4) { + /* non-Reordering has precedence over Reordering */ + ret_attrs = 4; /* nGnRE */ + } else if (s1lo == 8 || s2lo == 8) { + /* non-Gathering has precedence over Gathering */ + ret_attrs = 8; /* nGRE */ + } else { + ret_attrs = 0xc; /* GRE */ + } + } else { /* Normal memory */ + /* Outer/inner cacheability combine independently */ + ret_attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4 + | combine_cacheattr_nibble(s1lo, s2lo); + } + return ret_attrs; +} + +static uint8_t force_cacheattr_nibble_wb(uint8_t attr) +{ + /* + * Given the 4 bits specifying the outer or inner cacheability + * in MAIR format, return a value specifying Normal Write-Back, + * with the allocation and transient hints taken from the input + * if the input specified some kind of cacheable attribute. + */ + if (attr == 0 || attr == 4) { + /* + * 0 == an UNPREDICTABLE encoding + * 4 == Non-cacheable + * Either way, force Write-Back RW allocate non-transient + */ + return 0xf; + } + /* Change WriteThrough to WriteBack, keep allocation and transient hints */ + return attr | 4; +} + +/* + * Combine the memory type and cacheability attributes of + * s1 and s2 for the HCR_EL2.FWB == 1 case, returning the + * combined attributes in MAIR_EL1 format. + */ +static uint8_t combined_attrs_fwb(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) +{ + switch (s2.attrs) { + case 7: + /* Use stage 1 attributes */ + return s1.attrs; + case 6: + /* + * Force Normal Write-Back. Note that if S1 is Normal cacheable + * then we take the allocation hints from it; otherwise it is + * RW allocate, non-transient. + */ + if ((s1.attrs & 0xf0) == 0) { + /* S1 is Device */ + return 0xff; + } + /* Need to check the Inner and Outer nibbles separately */ + return force_cacheattr_nibble_wb(s1.attrs & 0xf) | + force_cacheattr_nibble_wb(s1.attrs >> 4) << 4; + case 5: + /* If S1 attrs are Device, use them; otherwise Normal Non-cacheable */ + if ((s1.attrs & 0xf0) == 0) { + return s1.attrs; + } + return 0x44; + case 0 ... 3: + /* Force Device, of subtype specified by S2 */ + return s2.attrs << 2; + default: + /* + * RESERVED values (including RES0 descriptor bit [5] being nonzero); + * arbitrarily force Device. + */ + return 0; + } +} + +/* + * Combine S1 and S2 cacheability/shareability attributes, per D4.5.4 + * and CombineS1S2Desc() + * + * @env: CPUARMState + * @s1: Attributes from stage 1 walk + * @s2: Attributes from stage 2 walk + */ +static ARMCacheAttrs combine_cacheattrs(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) +{ + ARMCacheAttrs ret; + bool tagged = false; + + assert(s2.is_s2_format && !s1.is_s2_format); + ret.is_s2_format = false; + + if (s1.attrs == 0xf0) { + tagged = true; + s1.attrs = 0xff; + } + + /* Combine shareability attributes (table D4-43) */ + if (s1.shareability == 2 || s2.shareability == 2) { + /* if either are outer-shareable, the result is outer-shareable */ + ret.shareability = 2; + } else if (s1.shareability == 3 || s2.shareability == 3) { + /* if either are inner-shareable, the result is inner-shareable */ + ret.shareability = 3; + } else { + /* both non-shareable */ + ret.shareability = 0; + } + + /* Combine memory type and cacheability attributes */ + if (arm_hcr_el2_eff(env) & HCR_FWB) { + ret.attrs = combined_attrs_fwb(env, s1, s2); + } else { + ret.attrs = combined_attrs_nofwb(env, s1, s2); + } + + /* + * Any location for which the resultant memory type is any + * type of Device memory is always treated as Outer Shareable. + * Any location for which the resultant memory type is Normal + * Inner Non-cacheable, Outer Non-cacheable is always treated + * as Outer Shareable. + * TODO: FEAT_XS adds another value (0x40) also meaning iNCoNC + */ + if ((ret.attrs & 0xf0) == 0 || ret.attrs == 0x44) { + ret.shareability = 2; + } + + /* TODO: CombineS1S2Desc does not consider transient, only WB, RWA. */ + if (tagged && ret.attrs == 0xff) { + ret.attrs = 0xf0; + } + + return ret; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index dd6fb93f33..b2dfe489bb 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -20,9 +20,6 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); -ARMCacheAttrs combine_cacheattrs(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2); - int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot); int simple_ap_to_rw_prot_is_user(int ap, bool is_user); From 3283222acd7579cdbc338360fa6e611027f3c8dc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 22/55] target/arm: Move get_phys_addr_lpae to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-16-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 416 +------------------------------------------- target/arm/ptw.c | 411 +++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 10 ++ 3 files changed, 429 insertions(+), 408 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index dab485e64a..7de815fe98 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10652,7 +10652,7 @@ int simple_ap_to_rw_prot_is_user(int ap, bool is_user) * @xn: XN (execute-never) bits * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0 */ -static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) +int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) { int prot = 0; @@ -10703,8 +10703,8 @@ static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) * @xn: XN (execute-never) bit * @pxn: PXN (privileged execute-never) bit */ -static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, - int ap, int ns, int xn, int pxn) +int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, + int ap, int ns, int xn, int pxn) { bool is_user = regime_is_user(env, mmu_idx); int prot_rw, user_rw; @@ -10919,8 +10919,8 @@ uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, * Returns true if the suggested S2 translation parameters are OK and * false otherwise. */ -static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, - int inputsize, int stride, int outputsize) +bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, + int inputsize, int stride, int outputsize) { const int grainsize = stride + 3; int startsizecheck; @@ -10980,7 +10980,7 @@ static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, #endif /* !CONFIG_USER_ONLY */ /* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ -static const uint8_t pamax_map[] = { +const uint8_t pamax_map[] = { [0] = 32, [1] = 36, [2] = 40, @@ -11159,8 +11159,8 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, } #ifndef CONFIG_USER_ONLY -static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, - ARMMMUIdx mmu_idx) +ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, + ARMMMUIdx mmu_idx) { uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; uint32_t el = regime_el(env, mmu_idx); @@ -11223,406 +11223,6 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, }; } -/** - * get_phys_addr_lpae: perform one stage of page table walk, LPAE format - * - * Returns false if the translation was successful. Otherwise, phys_ptr, attrs, - * prot and page_size may not be filled in, and the populated fsr value provides - * information on why the translation aborted, in the format of a long-format - * DFSR/IFSR fault register, with the following caveats: - * * the WnR bit is never set (the caller must do this). - * - * @env: CPUARMState - * @address: virtual address to get physical address for - * @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH - * @mmu_idx: MMU index indicating required translation regime - * @s1_is_el0: if @mmu_idx is ARMMMUIdx_Stage2 (so this is a stage 2 page table - * walk), must be true if this is stage 2 of a stage 1+2 walk for an - * EL0 access). If @mmu_idx is anything else, @s1_is_el0 is ignored. - * @phys_ptr: set to the physical address corresponding to the virtual address - * @attrs: set to the memory transaction attributes to use - * @prot: set to the permissions for the page containing phys_ptr - * @page_size_ptr: set to the size of the page containing phys_ptr - * @fi: set to fault info if the translation fails - * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes - */ -bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) -{ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - /* Read an LPAE long-descriptor translation table. */ - ARMFaultType fault_type = ARMFault_Translation; - uint32_t level; - ARMVAParameters param; - uint64_t ttbr; - hwaddr descaddr, indexmask, indexmask_grainsize; - uint32_t tableattrs; - target_ulong page_size; - uint32_t attrs; - int32_t stride; - int addrsize, inputsize, outputsize; - TCR *tcr = regime_tcr(env, mmu_idx); - int ap, ns, xn, pxn; - uint32_t el = regime_el(env, mmu_idx); - uint64_t descaddrmask; - bool aarch64 = arm_el_is_aa64(env, el); - bool guarded = false; - - /* TODO: This code does not support shareability levels. */ - if (aarch64) { - int ps; - - param = aa64_va_parameters(env, address, mmu_idx, - access_type != MMU_INST_FETCH); - level = 0; - - /* - * If TxSZ is programmed to a value larger than the maximum, - * or smaller than the effective minimum, it is IMPLEMENTATION - * DEFINED whether we behave as if the field were programmed - * within bounds, or if a level 0 Translation fault is generated. - * - * With FEAT_LVA, fault on less than minimum becomes required, - * so our choice is to always raise the fault. - */ - if (param.tsz_oob) { - fault_type = ARMFault_Translation; - goto do_fault; - } - - addrsize = 64 - 8 * param.tbi; - inputsize = 64 - param.tsz; - - /* - * Bound PS by PARANGE to find the effective output address size. - * ID_AA64MMFR0 is a read-only register so values outside of the - * supported mappings can be considered an implementation error. - */ - ps = FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); - ps = MIN(ps, param.ps); - assert(ps < ARRAY_SIZE(pamax_map)); - outputsize = pamax_map[ps]; - } else { - param = aa32_va_parameters(env, address, mmu_idx); - level = 1; - addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); - inputsize = addrsize - param.tsz; - outputsize = 40; - } - - /* - * We determined the region when collecting the parameters, but we - * have not yet validated that the address is valid for the region. - * Extract the top bits and verify that they all match select. - * - * For aa32, if inputsize == addrsize, then we have selected the - * region by exclusion in aa32_va_parameters and there is no more - * validation to do here. - */ - if (inputsize < addrsize) { - target_ulong top_bits = sextract64(address, inputsize, - addrsize - inputsize); - if (-top_bits != param.select) { - /* The gap between the two regions is a Translation fault */ - fault_type = ARMFault_Translation; - goto do_fault; - } - } - - if (param.using64k) { - stride = 13; - } else if (param.using16k) { - stride = 11; - } else { - stride = 9; - } - - /* Note that QEMU ignores shareability and cacheability attributes, - * so we don't need to do anything with the SH, ORGN, IRGN fields - * in the TTBCR. Similarly, TTBCR:A1 selects whether we get the - * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently - * implement any ASID-like capability so we can ignore it (instead - * we will always flush the TLB any time the ASID is changed). - */ - ttbr = regime_ttbr(env, mmu_idx, param.select); - - /* Here we should have set up all the parameters for the translation: - * inputsize, ttbr, epd, stride, tbi - */ - - if (param.epd) { - /* Translation table walk disabled => Translation fault on TLB miss - * Note: This is always 0 on 64-bit EL2 and EL3. - */ - goto do_fault; - } - - if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { - /* The starting level depends on the virtual address size (which can - * be up to 48 bits) and the translation granule size. It indicates - * the number of strides (stride bits at a time) needed to - * consume the bits of the input address. In the pseudocode this is: - * level = 4 - RoundUp((inputsize - grainsize) / stride) - * where their 'inputsize' is our 'inputsize', 'grainsize' is - * our 'stride + 3' and 'stride' is our 'stride'. - * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying: - * = 4 - (inputsize - stride - 3 + stride - 1) / stride - * = 4 - (inputsize - 4) / stride; - */ - level = 4 - (inputsize - 4) / stride; - } else { - /* For stage 2 translations the starting level is specified by the - * VTCR_EL2.SL0 field (whose interpretation depends on the page size) - */ - uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2); - uint32_t sl2 = extract64(tcr->raw_tcr, 33, 1); - uint32_t startlevel; - bool ok; - - /* SL2 is RES0 unless DS=1 & 4kb granule. */ - if (param.ds && stride == 9 && sl2) { - if (sl0 != 0) { - level = 0; - fault_type = ARMFault_Translation; - goto do_fault; - } - startlevel = -1; - } else if (!aarch64 || stride == 9) { - /* AArch32 or 4KB pages */ - startlevel = 2 - sl0; - - if (cpu_isar_feature(aa64_st, cpu)) { - startlevel &= 3; - } - } else { - /* 16KB or 64KB pages */ - startlevel = 3 - sl0; - } - - /* Check that the starting level is valid. */ - ok = check_s2_mmu_setup(cpu, aarch64, startlevel, - inputsize, stride, outputsize); - if (!ok) { - fault_type = ARMFault_Translation; - goto do_fault; - } - level = startlevel; - } - - indexmask_grainsize = MAKE_64BIT_MASK(0, stride + 3); - indexmask = MAKE_64BIT_MASK(0, inputsize - (stride * (4 - level))); - - /* Now we can extract the actual base address from the TTBR */ - descaddr = extract64(ttbr, 0, 48); - - /* - * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [5:2] of TTBR. - * - * Otherwise, if the base address is out of range, raise AddressSizeFault. - * In the pseudocode, this is !IsZero(baseregister<47:outputsize>), - * but we've just cleared the bits above 47, so simplify the test. - */ - if (outputsize > 48) { - descaddr |= extract64(ttbr, 2, 4) << 48; - } else if (descaddr >> outputsize) { - level = 0; - fault_type = ARMFault_AddressSize; - goto do_fault; - } - - /* - * We rely on this masking to clear the RES0 bits at the bottom of the TTBR - * and also to mask out CnP (bit 0) which could validly be non-zero. - */ - descaddr &= ~indexmask; - - /* - * For AArch32, the address field in the descriptor goes up to bit 39 - * for both v7 and v8. However, for v8 the SBZ bits [47:40] must be 0 - * or an AddressSize fault is raised. So for v8 we extract those SBZ - * bits as part of the address, which will be checked via outputsize. - * For AArch64, the address field goes up to bit 47, or 49 with FEAT_LPA2; - * the highest bits of a 52-bit output are placed elsewhere. - */ - if (param.ds) { - descaddrmask = MAKE_64BIT_MASK(0, 50); - } else if (arm_feature(env, ARM_FEATURE_V8)) { - descaddrmask = MAKE_64BIT_MASK(0, 48); - } else { - descaddrmask = MAKE_64BIT_MASK(0, 40); - } - descaddrmask &= ~indexmask_grainsize; - - /* Secure accesses start with the page table in secure memory and - * can be downgraded to non-secure at any step. Non-secure accesses - * remain non-secure. We implement this by just ORing in the NSTable/NS - * bits at each step. - */ - tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4); - for (;;) { - uint64_t descriptor; - bool nstable; - - descaddr |= (address >> (stride * (4 - level))) & indexmask; - descaddr &= ~7ULL; - nstable = extract32(tableattrs, 4, 1); - descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - - if (!(descriptor & 1) || - (!(descriptor & 2) && (level == 3))) { - /* Invalid, or the Reserved level 3 encoding */ - goto do_fault; - } - - descaddr = descriptor & descaddrmask; - - /* - * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [15:12] - * of descriptor. For FEAT_LPA2 and effective DS, bits [51:50] of - * descaddr are in [9:8]. Otherwise, if descaddr is out of range, - * raise AddressSizeFault. - */ - if (outputsize > 48) { - if (param.ds) { - descaddr |= extract64(descriptor, 8, 2) << 50; - } else { - descaddr |= extract64(descriptor, 12, 4) << 48; - } - } else if (descaddr >> outputsize) { - fault_type = ARMFault_AddressSize; - goto do_fault; - } - - if ((descriptor & 2) && (level < 3)) { - /* Table entry. The top five bits are attributes which may - * propagate down through lower levels of the table (and - * which are all arranged so that 0 means "no effect", so - * we can gather them up by ORing in the bits at each level). - */ - tableattrs |= extract64(descriptor, 59, 5); - level++; - indexmask = indexmask_grainsize; - continue; - } - /* - * Block entry at level 1 or 2, or page entry at level 3. - * These are basically the same thing, although the number - * of bits we pull in from the vaddr varies. Note that although - * descaddrmask masks enough of the low bits of the descriptor - * to give a correct page or table address, the address field - * in a block descriptor is smaller; so we need to explicitly - * clear the lower bits here before ORing in the low vaddr bits. - */ - page_size = (1ULL << ((stride * (4 - level)) + 3)); - descaddr &= ~(page_size - 1); - descaddr |= (address & (page_size - 1)); - /* Extract attributes from the descriptor */ - attrs = extract64(descriptor, 2, 10) - | (extract64(descriptor, 52, 12) << 10); - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - /* Stage 2 table descriptors do not include any attribute fields */ - break; - } - /* Merge in attributes from table descriptors */ - attrs |= nstable << 3; /* NS */ - guarded = extract64(descriptor, 50, 1); /* GP */ - if (param.hpd) { - /* HPD disables all the table attributes except NSTable. */ - break; - } - attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */ - /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1 - * means "force PL1 access only", which means forcing AP[1] to 0. - */ - attrs &= ~(extract32(tableattrs, 2, 1) << 4); /* !APT[0] => AP[1] */ - attrs |= extract32(tableattrs, 3, 1) << 5; /* APT[1] => AP[2] */ - break; - } - /* Here descaddr is the final physical address, and attributes - * are all in attrs. - */ - fault_type = ARMFault_AccessFlag; - if ((attrs & (1 << 8)) == 0) { - /* Access flag */ - goto do_fault; - } - - ap = extract32(attrs, 4, 2); - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - ns = mmu_idx == ARMMMUIdx_Stage2; - xn = extract32(attrs, 11, 2); - *prot = get_S2prot(env, ap, xn, s1_is_el0); - } else { - ns = extract32(attrs, 3, 1); - xn = extract32(attrs, 12, 1); - pxn = extract32(attrs, 11, 1); - *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn); - } - - fault_type = ARMFault_Permission; - if (!(*prot & (1 << access_type))) { - goto do_fault; - } - - if (ns) { - /* The NS bit will (as required by the architecture) have no effect if - * the CPU doesn't support TZ or this is a non-secure translation - * regime, because the attribute will already be non-secure. - */ - txattrs->secure = false; - } - /* When in aarch64 mode, and BTI is enabled, remember GP in the IOTLB. */ - if (aarch64 && guarded && cpu_isar_feature(aa64_bti, cpu)) { - arm_tlb_bti_gp(txattrs) = true; - } - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - cacheattrs->is_s2_format = true; - cacheattrs->attrs = extract32(attrs, 0, 4); - } else { - /* Index into MAIR registers for cache attributes */ - uint8_t attrindx = extract32(attrs, 0, 3); - uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)]; - assert(attrindx <= 7); - cacheattrs->is_s2_format = false; - cacheattrs->attrs = extract64(mair, attrindx * 8, 8); - } - - /* - * For FEAT_LPA2 and effective DS, the SH field in the attributes - * was re-purposed for output address bits. The SH attribute in - * that case comes from TCR_ELx, which we extracted earlier. - */ - if (param.ds) { - cacheattrs->shareability = param.sh; - } else { - cacheattrs->shareability = extract32(attrs, 6, 2); - } - - *phys_ptr = descaddr; - *page_size_ptr = page_size; - return false; - -do_fault: - fi->type = fault_type; - fi->level = level; - /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ - fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2 || - mmu_idx == ARMMMUIdx_Stage2_S); - fi->s1ns = mmu_idx == ARMMMUIdx_Stage2; - return true; -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index f2ca2bb8fe..cbccf91b13 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -314,6 +314,417 @@ do_fault: return true; } +/** + * get_phys_addr_lpae: perform one stage of page table walk, LPAE format + * + * Returns false if the translation was successful. Otherwise, phys_ptr, + * attrs, prot and page_size may not be filled in, and the populated fsr + * value provides information on why the translation aborted, in the format + * of a long-format DFSR/IFSR fault register, with the following caveat: + * the WnR bit is never set (the caller must do this). + * + * @env: CPUARMState + * @address: virtual address to get physical address for + * @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH + * @mmu_idx: MMU index indicating required translation regime + * @s1_is_el0: if @mmu_idx is ARMMMUIdx_Stage2 (so this is a stage 2 page + * table walk), must be true if this is stage 2 of a stage 1+2 + * walk for an EL0 access. If @mmu_idx is anything else, + * @s1_is_el0 is ignored. + * @phys_ptr: set to the physical address corresponding to the virtual address + * @attrs: set to the memory transaction attributes to use + * @prot: set to the permissions for the page containing phys_ptr + * @page_size_ptr: set to the size of the page containing phys_ptr + * @fi: set to fault info if the translation fails + * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes + */ +bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, + hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +{ + ARMCPU *cpu = env_archcpu(env); + CPUState *cs = CPU(cpu); + /* Read an LPAE long-descriptor translation table. */ + ARMFaultType fault_type = ARMFault_Translation; + uint32_t level; + ARMVAParameters param; + uint64_t ttbr; + hwaddr descaddr, indexmask, indexmask_grainsize; + uint32_t tableattrs; + target_ulong page_size; + uint32_t attrs; + int32_t stride; + int addrsize, inputsize, outputsize; + TCR *tcr = regime_tcr(env, mmu_idx); + int ap, ns, xn, pxn; + uint32_t el = regime_el(env, mmu_idx); + uint64_t descaddrmask; + bool aarch64 = arm_el_is_aa64(env, el); + bool guarded = false; + + /* TODO: This code does not support shareability levels. */ + if (aarch64) { + int ps; + + param = aa64_va_parameters(env, address, mmu_idx, + access_type != MMU_INST_FETCH); + level = 0; + + /* + * If TxSZ is programmed to a value larger than the maximum, + * or smaller than the effective minimum, it is IMPLEMENTATION + * DEFINED whether we behave as if the field were programmed + * within bounds, or if a level 0 Translation fault is generated. + * + * With FEAT_LVA, fault on less than minimum becomes required, + * so our choice is to always raise the fault. + */ + if (param.tsz_oob) { + fault_type = ARMFault_Translation; + goto do_fault; + } + + addrsize = 64 - 8 * param.tbi; + inputsize = 64 - param.tsz; + + /* + * Bound PS by PARANGE to find the effective output address size. + * ID_AA64MMFR0 is a read-only register so values outside of the + * supported mappings can be considered an implementation error. + */ + ps = FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); + ps = MIN(ps, param.ps); + assert(ps < ARRAY_SIZE(pamax_map)); + outputsize = pamax_map[ps]; + } else { + param = aa32_va_parameters(env, address, mmu_idx); + level = 1; + addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); + inputsize = addrsize - param.tsz; + outputsize = 40; + } + + /* + * We determined the region when collecting the parameters, but we + * have not yet validated that the address is valid for the region. + * Extract the top bits and verify that they all match select. + * + * For aa32, if inputsize == addrsize, then we have selected the + * region by exclusion in aa32_va_parameters and there is no more + * validation to do here. + */ + if (inputsize < addrsize) { + target_ulong top_bits = sextract64(address, inputsize, + addrsize - inputsize); + if (-top_bits != param.select) { + /* The gap between the two regions is a Translation fault */ + fault_type = ARMFault_Translation; + goto do_fault; + } + } + + if (param.using64k) { + stride = 13; + } else if (param.using16k) { + stride = 11; + } else { + stride = 9; + } + + /* + * Note that QEMU ignores shareability and cacheability attributes, + * so we don't need to do anything with the SH, ORGN, IRGN fields + * in the TTBCR. Similarly, TTBCR:A1 selects whether we get the + * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently + * implement any ASID-like capability so we can ignore it (instead + * we will always flush the TLB any time the ASID is changed). + */ + ttbr = regime_ttbr(env, mmu_idx, param.select); + + /* + * Here we should have set up all the parameters for the translation: + * inputsize, ttbr, epd, stride, tbi + */ + + if (param.epd) { + /* + * Translation table walk disabled => Translation fault on TLB miss + * Note: This is always 0 on 64-bit EL2 and EL3. + */ + goto do_fault; + } + + if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { + /* + * The starting level depends on the virtual address size (which can + * be up to 48 bits) and the translation granule size. It indicates + * the number of strides (stride bits at a time) needed to + * consume the bits of the input address. In the pseudocode this is: + * level = 4 - RoundUp((inputsize - grainsize) / stride) + * where their 'inputsize' is our 'inputsize', 'grainsize' is + * our 'stride + 3' and 'stride' is our 'stride'. + * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying: + * = 4 - (inputsize - stride - 3 + stride - 1) / stride + * = 4 - (inputsize - 4) / stride; + */ + level = 4 - (inputsize - 4) / stride; + } else { + /* + * For stage 2 translations the starting level is specified by the + * VTCR_EL2.SL0 field (whose interpretation depends on the page size) + */ + uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2); + uint32_t sl2 = extract64(tcr->raw_tcr, 33, 1); + uint32_t startlevel; + bool ok; + + /* SL2 is RES0 unless DS=1 & 4kb granule. */ + if (param.ds && stride == 9 && sl2) { + if (sl0 != 0) { + level = 0; + fault_type = ARMFault_Translation; + goto do_fault; + } + startlevel = -1; + } else if (!aarch64 || stride == 9) { + /* AArch32 or 4KB pages */ + startlevel = 2 - sl0; + + if (cpu_isar_feature(aa64_st, cpu)) { + startlevel &= 3; + } + } else { + /* 16KB or 64KB pages */ + startlevel = 3 - sl0; + } + + /* Check that the starting level is valid. */ + ok = check_s2_mmu_setup(cpu, aarch64, startlevel, + inputsize, stride, outputsize); + if (!ok) { + fault_type = ARMFault_Translation; + goto do_fault; + } + level = startlevel; + } + + indexmask_grainsize = MAKE_64BIT_MASK(0, stride + 3); + indexmask = MAKE_64BIT_MASK(0, inputsize - (stride * (4 - level))); + + /* Now we can extract the actual base address from the TTBR */ + descaddr = extract64(ttbr, 0, 48); + + /* + * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [5:2] of TTBR. + * + * Otherwise, if the base address is out of range, raise AddressSizeFault. + * In the pseudocode, this is !IsZero(baseregister<47:outputsize>), + * but we've just cleared the bits above 47, so simplify the test. + */ + if (outputsize > 48) { + descaddr |= extract64(ttbr, 2, 4) << 48; + } else if (descaddr >> outputsize) { + level = 0; + fault_type = ARMFault_AddressSize; + goto do_fault; + } + + /* + * We rely on this masking to clear the RES0 bits at the bottom of the TTBR + * and also to mask out CnP (bit 0) which could validly be non-zero. + */ + descaddr &= ~indexmask; + + /* + * For AArch32, the address field in the descriptor goes up to bit 39 + * for both v7 and v8. However, for v8 the SBZ bits [47:40] must be 0 + * or an AddressSize fault is raised. So for v8 we extract those SBZ + * bits as part of the address, which will be checked via outputsize. + * For AArch64, the address field goes up to bit 47, or 49 with FEAT_LPA2; + * the highest bits of a 52-bit output are placed elsewhere. + */ + if (param.ds) { + descaddrmask = MAKE_64BIT_MASK(0, 50); + } else if (arm_feature(env, ARM_FEATURE_V8)) { + descaddrmask = MAKE_64BIT_MASK(0, 48); + } else { + descaddrmask = MAKE_64BIT_MASK(0, 40); + } + descaddrmask &= ~indexmask_grainsize; + + /* + * Secure accesses start with the page table in secure memory and + * can be downgraded to non-secure at any step. Non-secure accesses + * remain non-secure. We implement this by just ORing in the NSTable/NS + * bits at each step. + */ + tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4); + for (;;) { + uint64_t descriptor; + bool nstable; + + descaddr |= (address >> (stride * (4 - level))) & indexmask; + descaddr &= ~7ULL; + nstable = extract32(tableattrs, 4, 1); + descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + + if (!(descriptor & 1) || + (!(descriptor & 2) && (level == 3))) { + /* Invalid, or the Reserved level 3 encoding */ + goto do_fault; + } + + descaddr = descriptor & descaddrmask; + + /* + * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [15:12] + * of descriptor. For FEAT_LPA2 and effective DS, bits [51:50] of + * descaddr are in [9:8]. Otherwise, if descaddr is out of range, + * raise AddressSizeFault. + */ + if (outputsize > 48) { + if (param.ds) { + descaddr |= extract64(descriptor, 8, 2) << 50; + } else { + descaddr |= extract64(descriptor, 12, 4) << 48; + } + } else if (descaddr >> outputsize) { + fault_type = ARMFault_AddressSize; + goto do_fault; + } + + if ((descriptor & 2) && (level < 3)) { + /* + * Table entry. The top five bits are attributes which may + * propagate down through lower levels of the table (and + * which are all arranged so that 0 means "no effect", so + * we can gather them up by ORing in the bits at each level). + */ + tableattrs |= extract64(descriptor, 59, 5); + level++; + indexmask = indexmask_grainsize; + continue; + } + /* + * Block entry at level 1 or 2, or page entry at level 3. + * These are basically the same thing, although the number + * of bits we pull in from the vaddr varies. Note that although + * descaddrmask masks enough of the low bits of the descriptor + * to give a correct page or table address, the address field + * in a block descriptor is smaller; so we need to explicitly + * clear the lower bits here before ORing in the low vaddr bits. + */ + page_size = (1ULL << ((stride * (4 - level)) + 3)); + descaddr &= ~(page_size - 1); + descaddr |= (address & (page_size - 1)); + /* Extract attributes from the descriptor */ + attrs = extract64(descriptor, 2, 10) + | (extract64(descriptor, 52, 12) << 10); + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + /* Stage 2 table descriptors do not include any attribute fields */ + break; + } + /* Merge in attributes from table descriptors */ + attrs |= nstable << 3; /* NS */ + guarded = extract64(descriptor, 50, 1); /* GP */ + if (param.hpd) { + /* HPD disables all the table attributes except NSTable. */ + break; + } + attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */ + /* + * The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1 + * means "force PL1 access only", which means forcing AP[1] to 0. + */ + attrs &= ~(extract32(tableattrs, 2, 1) << 4); /* !APT[0] => AP[1] */ + attrs |= extract32(tableattrs, 3, 1) << 5; /* APT[1] => AP[2] */ + break; + } + /* + * Here descaddr is the final physical address, and attributes + * are all in attrs. + */ + fault_type = ARMFault_AccessFlag; + if ((attrs & (1 << 8)) == 0) { + /* Access flag */ + goto do_fault; + } + + ap = extract32(attrs, 4, 2); + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + ns = mmu_idx == ARMMMUIdx_Stage2; + xn = extract32(attrs, 11, 2); + *prot = get_S2prot(env, ap, xn, s1_is_el0); + } else { + ns = extract32(attrs, 3, 1); + xn = extract32(attrs, 12, 1); + pxn = extract32(attrs, 11, 1); + *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn); + } + + fault_type = ARMFault_Permission; + if (!(*prot & (1 << access_type))) { + goto do_fault; + } + + if (ns) { + /* + * The NS bit will (as required by the architecture) have no effect if + * the CPU doesn't support TZ or this is a non-secure translation + * regime, because the attribute will already be non-secure. + */ + txattrs->secure = false; + } + /* When in aarch64 mode, and BTI is enabled, remember GP in the IOTLB. */ + if (aarch64 && guarded && cpu_isar_feature(aa64_bti, cpu)) { + arm_tlb_bti_gp(txattrs) = true; + } + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + cacheattrs->is_s2_format = true; + cacheattrs->attrs = extract32(attrs, 0, 4); + } else { + /* Index into MAIR registers for cache attributes */ + uint8_t attrindx = extract32(attrs, 0, 3); + uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)]; + assert(attrindx <= 7); + cacheattrs->is_s2_format = false; + cacheattrs->attrs = extract64(mair, attrindx * 8, 8); + } + + /* + * For FEAT_LPA2 and effective DS, the SH field in the attributes + * was re-purposed for output address bits. The SH attribute in + * that case comes from TCR_ELx, which we extracted earlier. + */ + if (param.ds) { + cacheattrs->shareability = param.sh; + } else { + cacheattrs->shareability = extract32(attrs, 6, 2); + } + + *phys_ptr = descaddr; + *page_size_ptr = page_size; + return false; + +do_fault: + fi->type = fault_type; + fi->level = level; + /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ + fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2 || + mmu_idx == ARMMMUIdx_Stage2_S); + fi->s1ns = mmu_idx == ARMMMUIdx_Stage2; + return true; +} + static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index b2dfe489bb..31744df664 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,6 +11,8 @@ #ifndef CONFIG_USER_ONLY +extern const uint8_t pamax_map[7]; + uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, @@ -30,6 +32,14 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } +ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, + ARMMMUIdx mmu_idx); +bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, + int inputsize, int stride, int outputsize); +int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0); +int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, + int ap, int ns, int xn, int pxn); + bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From 11552bb0d95484c99233ec2f09c25261885d08ed Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 23/55] target/arm: Move arm_{ldl,ldq}_ptw to ptw.c Move the ptw load functions, plus 3 common subroutines: S1_ptw_translate, ptw_attrs_are_device, and regime_translation_big_endian. This also allows get_phys_addr_lpae to become static again. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-17-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 141 -------------------------------------- target/arm/ptw.c | 160 ++++++++++++++++++++++++++++++++++++++++++-- target/arm/ptw.h | 13 ---- 3 files changed, 154 insertions(+), 160 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 7de815fe98..398bcd62ab 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10475,12 +10475,6 @@ bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; } -static inline bool regime_translation_big_endian(CPUARMState *env, - ARMMMUIdx mmu_idx) -{ - return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; -} - /* Return the TTBR associated with this translation regime */ uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { @@ -10773,141 +10767,6 @@ int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } -static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) -{ - /* - * For an S1 page table walk, the stage 1 attributes are always - * some form of "this is Normal memory". The combined S1+S2 - * attributes are therefore only Device if stage 2 specifies Device. - * With HCR_EL2.FWB == 0 this is when descriptor bits [5:4] are 0b00, - * ie when cacheattrs.attrs bits [3:2] are 0b00. - * With HCR_EL2.FWB == 1 this is when descriptor bit [4] is 0, ie - * when cacheattrs.attrs bit [2] is 0. - */ - assert(cacheattrs.is_s2_format); - if (arm_hcr_el2_eff(env) & HCR_FWB) { - return (cacheattrs.attrs & 0x4) == 0; - } else { - return (cacheattrs.attrs & 0xc) == 0; - } -} - -/* Translate a S1 pagetable walk through S2 if needed. */ -static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, - hwaddr addr, bool *is_secure, - ARMMMUFaultInfo *fi) -{ - if (arm_mmu_idx_is_stage1_of_2(mmu_idx) && - !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { - target_ulong s2size; - hwaddr s2pa; - int s2prot; - int ret; - ARMMMUIdx s2_mmu_idx = *is_secure ? ARMMMUIdx_Stage2_S - : ARMMMUIdx_Stage2; - ARMCacheAttrs cacheattrs = {}; - MemTxAttrs txattrs = {}; - - ret = get_phys_addr_lpae(env, addr, MMU_DATA_LOAD, s2_mmu_idx, false, - &s2pa, &txattrs, &s2prot, &s2size, fi, - &cacheattrs); - if (ret) { - assert(fi->type != ARMFault_None); - fi->s2addr = addr; - fi->stage2 = true; - fi->s1ptw = true; - fi->s1ns = !*is_secure; - return ~0; - } - if ((arm_hcr_el2_eff(env) & HCR_PTW) && - ptw_attrs_are_device(env, cacheattrs)) { - /* - * PTW set and S1 walk touched S2 Device memory: - * generate Permission fault. - */ - fi->type = ARMFault_Permission; - fi->s2addr = addr; - fi->stage2 = true; - fi->s1ptw = true; - fi->s1ns = !*is_secure; - return ~0; - } - - if (arm_is_secure_below_el3(env)) { - /* Check if page table walk is to secure or non-secure PA space. */ - if (*is_secure) { - *is_secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); - } else { - *is_secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); - } - } else { - assert(!*is_secure); - } - - addr = s2pa; - } - return addr; -} - -/* All loads done in the course of a page table walk go through here. */ -uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) -{ - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - MemTxAttrs attrs = {}; - MemTxResult result = MEMTX_OK; - AddressSpace *as; - uint32_t data; - - addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); - attrs.secure = is_secure; - as = arm_addressspace(cs, attrs); - if (fi->s1ptw) { - return 0; - } - if (regime_translation_big_endian(env, mmu_idx)) { - data = address_space_ldl_be(as, addr, attrs, &result); - } else { - data = address_space_ldl_le(as, addr, attrs, &result); - } - if (result == MEMTX_OK) { - return data; - } - fi->type = ARMFault_SyncExternalOnWalk; - fi->ea = arm_extabort_type(result); - return 0; -} - -uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) -{ - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - MemTxAttrs attrs = {}; - MemTxResult result = MEMTX_OK; - AddressSpace *as; - uint64_t data; - - addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); - attrs.secure = is_secure; - as = arm_addressspace(cs, attrs); - if (fi->s1ptw) { - return 0; - } - if (regime_translation_big_endian(env, mmu_idx)) { - data = address_space_ldq_be(as, addr, attrs, &result); - } else { - data = address_space_ldq_le(as, addr, attrs, &result); - } - if (result == MEMTX_OK) { - return data; - } - fi->type = ARMFault_SyncExternalOnWalk; - fi->ea = arm_extabort_type(result); - return 0; -} - /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.c b/target/arm/ptw.c index cbccf91b13..e4b860d2ae 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -15,6 +15,154 @@ #include "ptw.h" +static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, hwaddr *phys_ptr, + MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) + __attribute__((nonnull)); + +static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; +} + +static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) +{ + /* + * For an S1 page table walk, the stage 1 attributes are always + * some form of "this is Normal memory". The combined S1+S2 + * attributes are therefore only Device if stage 2 specifies Device. + * With HCR_EL2.FWB == 0 this is when descriptor bits [5:4] are 0b00, + * ie when cacheattrs.attrs bits [3:2] are 0b00. + * With HCR_EL2.FWB == 1 this is when descriptor bit [4] is 0, ie + * when cacheattrs.attrs bit [2] is 0. + */ + assert(cacheattrs.is_s2_format); + if (arm_hcr_el2_eff(env) & HCR_FWB) { + return (cacheattrs.attrs & 0x4) == 0; + } else { + return (cacheattrs.attrs & 0xc) == 0; + } +} + +/* Translate a S1 pagetable walk through S2 if needed. */ +static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, + hwaddr addr, bool *is_secure, + ARMMMUFaultInfo *fi) +{ + if (arm_mmu_idx_is_stage1_of_2(mmu_idx) && + !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { + target_ulong s2size; + hwaddr s2pa; + int s2prot; + int ret; + ARMMMUIdx s2_mmu_idx = *is_secure ? ARMMMUIdx_Stage2_S + : ARMMMUIdx_Stage2; + ARMCacheAttrs cacheattrs = {}; + MemTxAttrs txattrs = {}; + + ret = get_phys_addr_lpae(env, addr, MMU_DATA_LOAD, s2_mmu_idx, false, + &s2pa, &txattrs, &s2prot, &s2size, fi, + &cacheattrs); + if (ret) { + assert(fi->type != ARMFault_None); + fi->s2addr = addr; + fi->stage2 = true; + fi->s1ptw = true; + fi->s1ns = !*is_secure; + return ~0; + } + if ((arm_hcr_el2_eff(env) & HCR_PTW) && + ptw_attrs_are_device(env, cacheattrs)) { + /* + * PTW set and S1 walk touched S2 Device memory: + * generate Permission fault. + */ + fi->type = ARMFault_Permission; + fi->s2addr = addr; + fi->stage2 = true; + fi->s1ptw = true; + fi->s1ns = !*is_secure; + return ~0; + } + + if (arm_is_secure_below_el3(env)) { + /* Check if page table walk is to secure or non-secure PA space. */ + if (*is_secure) { + *is_secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); + } else { + *is_secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); + } + } else { + assert(!*is_secure); + } + + addr = s2pa; + } + return addr; +} + +/* All loads done in the course of a page table walk go through here. */ +static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + MemTxAttrs attrs = {}; + MemTxResult result = MEMTX_OK; + AddressSpace *as; + uint32_t data; + + addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); + attrs.secure = is_secure; + as = arm_addressspace(cs, attrs); + if (fi->s1ptw) { + return 0; + } + if (regime_translation_big_endian(env, mmu_idx)) { + data = address_space_ldl_be(as, addr, attrs, &result); + } else { + data = address_space_ldl_le(as, addr, attrs, &result); + } + if (result == MEMTX_OK) { + return data; + } + fi->type = ARMFault_SyncExternalOnWalk; + fi->ea = arm_extabort_type(result); + return 0; +} + +static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + MemTxAttrs attrs = {}; + MemTxResult result = MEMTX_OK; + AddressSpace *as; + uint64_t data; + + addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); + attrs.secure = is_secure; + as = arm_addressspace(cs, attrs); + if (fi->s1ptw) { + return 0; + } + if (regime_translation_big_endian(env, mmu_idx)) { + data = address_space_ldq_be(as, addr, attrs, &result); + } else { + data = address_space_ldq_le(as, addr, attrs, &result); + } + if (result == MEMTX_OK) { + return data; + } + fi->type = ARMFault_SyncExternalOnWalk; + fi->ea = arm_extabort_type(result); + return 0; +} + static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, uint32_t *table, uint32_t address) { @@ -338,12 +486,12 @@ do_fault: * @fi: set to fault info if the translation fails * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes */ -bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, hwaddr *phys_ptr, + MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 31744df664..28b8cb9fb8 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -13,11 +13,6 @@ extern const uint8_t pamax_map[7]; -uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); -uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); - bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); @@ -40,13 +35,5 @@ int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0); int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, int ap, int ns, int xn, int pxn); -bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) - __attribute__((nonnull)); - #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From cd6bc4d51730e9cf47489029d078d18c3bcb3ae2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 24/55] target/arm: Move {arm_s1_, }regime_using_lpae_format to tlb_helper.c These functions are used for both page table walking and for deciding what format in which to deliver exception results. Since ptw.c is only present for system mode, put the functions into tlb_helper.c. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-18-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 24 ------------------------ target/arm/tlb_helper.c | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 398bcd62ab..d2b196ff3e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10515,30 +10515,6 @@ ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) } #endif /* !CONFIG_USER_ONLY */ -/* Return true if the translation regime is using LPAE format page tables */ -bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - int el = regime_el(env, mmu_idx); - if (el == 2 || arm_el_is_aa64(env, el)) { - return true; - } - if (arm_feature(env, ARM_FEATURE_LPAE) - && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) { - return true; - } - return false; -} - -/* Returns true if the stage 1 translation regime is using LPAE format page - * tables. Used when raising alignment exceptions, whose FSR changes depending - * on whether the long or short descriptor format is in use. */ -bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - mmu_idx = stage_1_mmu_idx(mmu_idx); - - return regime_using_lpae_format(env, mmu_idx); -} - #ifndef CONFIG_USER_ONLY bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { diff --git a/target/arm/tlb_helper.c b/target/arm/tlb_helper.c index 6421e16202..7d8a86b3c4 100644 --- a/target/arm/tlb_helper.c +++ b/target/arm/tlb_helper.c @@ -11,6 +11,32 @@ #include "exec/exec-all.h" #include "exec/helper-proto.h" + +/* Return true if the translation regime is using LPAE format page tables */ +bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + int el = regime_el(env, mmu_idx); + if (el == 2 || arm_el_is_aa64(env, el)) { + return true; + } + if (arm_feature(env, ARM_FEATURE_LPAE) + && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) { + return true; + } + return false; +} + +/* + * Returns true if the stage 1 translation regime is using LPAE format page + * tables. Used when raising alignment exceptions, whose FSR changes depending + * on whether the long or short descriptor format is in use. + */ +bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + mmu_idx = stage_1_mmu_idx(mmu_idx); + return regime_using_lpae_format(env, mmu_idx); +} + static inline uint32_t merge_syn_data_abort(uint32_t template_syn, unsigned int target_el, bool same_el, bool ea, From 1c73d84807712aad4c7b2e4e753d6870ab81145d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 25/55] target/arm: Move arm_pamax, pamax_map into ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-19-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 25 ------------------------- target/arm/ptw.c | 25 +++++++++++++++++++++++++ target/arm/ptw.h | 2 -- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d2b196ff3e..563e34ecde 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10814,31 +10814,6 @@ bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, } #endif /* !CONFIG_USER_ONLY */ -/* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ -const uint8_t pamax_map[] = { - [0] = 32, - [1] = 36, - [2] = 40, - [3] = 42, - [4] = 44, - [5] = 48, - [6] = 52, -}; - -/* The cpu-specific constant value of PAMax; also used by hw/arm/virt. */ -unsigned int arm_pamax(ARMCPU *cpu) -{ - unsigned int parange = - FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); - - /* - * id_aa64mmfr0 is a read-only register so values outside of the - * supported mappings can be considered an implementation error. - */ - assert(parange < ARRAY_SIZE(pamax_map)); - return pamax_map[parange]; -} - int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index e4b860d2ae..d754273fa1 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -23,6 +23,31 @@ static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) __attribute__((nonnull)); +/* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ +static const uint8_t pamax_map[] = { + [0] = 32, + [1] = 36, + [2] = 40, + [3] = 42, + [4] = 44, + [5] = 48, + [6] = 52, +}; + +/* The cpu-specific constant value of PAMax; also used by hw/arm/virt. */ +unsigned int arm_pamax(ARMCPU *cpu) +{ + unsigned int parange = + FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); + + /* + * id_aa64mmfr0 is a read-only register so values outside of the + * supported mappings can be considered an implementation error. + */ + assert(parange < ARRAY_SIZE(pamax_map)); + return pamax_map[parange]; +} + static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) { return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 28b8cb9fb8..fba650d01c 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,8 +11,6 @@ #ifndef CONFIG_USER_ONLY -extern const uint8_t pamax_map[7]; - bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); From f8526edc2f32b66a585618afc1dabd960a4c9b95 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 26/55] target/arm: Move get_S1prot, get_S2prot to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-20-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 128 -------------------------------------------- target/arm/ptw.c | 128 ++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 -- 3 files changed, 128 insertions(+), 131 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 563e34ecde..148eb28ba8 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10615,134 +10615,6 @@ int simple_ap_to_rw_prot_is_user(int ap, bool is_user) } } -/* Translate S2 section/page access permissions to protection flags - * - * @env: CPUARMState - * @s2ap: The 2-bit stage2 access permissions (S2AP) - * @xn: XN (execute-never) bits - * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0 - */ -int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) -{ - int prot = 0; - - if (s2ap & 1) { - prot |= PAGE_READ; - } - if (s2ap & 2) { - prot |= PAGE_WRITE; - } - - if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) { - switch (xn) { - case 0: - prot |= PAGE_EXEC; - break; - case 1: - if (s1_is_el0) { - prot |= PAGE_EXEC; - } - break; - case 2: - break; - case 3: - if (!s1_is_el0) { - prot |= PAGE_EXEC; - } - break; - default: - g_assert_not_reached(); - } - } else { - if (!extract32(xn, 1, 1)) { - if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) { - prot |= PAGE_EXEC; - } - } - } - return prot; -} - -/* Translate section/page access permissions to protection flags - * - * @env: CPUARMState - * @mmu_idx: MMU index indicating required translation regime - * @is_aa64: TRUE if AArch64 - * @ap: The 2-bit simple AP (AP[2:1]) - * @ns: NS (non-secure) bit - * @xn: XN (execute-never) bit - * @pxn: PXN (privileged execute-never) bit - */ -int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, - int ap, int ns, int xn, int pxn) -{ - bool is_user = regime_is_user(env, mmu_idx); - int prot_rw, user_rw; - bool have_wxn; - int wxn = 0; - - assert(mmu_idx != ARMMMUIdx_Stage2); - assert(mmu_idx != ARMMMUIdx_Stage2_S); - - user_rw = simple_ap_to_rw_prot_is_user(ap, true); - if (is_user) { - prot_rw = user_rw; - } else { - if (user_rw && regime_is_pan(env, mmu_idx)) { - /* PAN forbids data accesses but doesn't affect insn fetch */ - prot_rw = 0; - } else { - prot_rw = simple_ap_to_rw_prot_is_user(ap, false); - } - } - - if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) { - return prot_rw; - } - - /* TODO have_wxn should be replaced with - * ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2) - * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE - * compatible processors have EL2, which is required for [U]WXN. - */ - have_wxn = arm_feature(env, ARM_FEATURE_LPAE); - - if (have_wxn) { - wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN; - } - - if (is_aa64) { - if (regime_has_2_ranges(mmu_idx) && !is_user) { - xn = pxn || (user_rw & PAGE_WRITE); - } - } else if (arm_feature(env, ARM_FEATURE_V7)) { - switch (regime_el(env, mmu_idx)) { - case 1: - case 3: - if (is_user) { - xn = xn || !(user_rw & PAGE_READ); - } else { - int uwxn = 0; - if (have_wxn) { - uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN; - } - xn = xn || !(prot_rw & PAGE_READ) || pxn || - (uwxn && (user_rw & PAGE_WRITE)); - } - break; - case 2: - break; - } - } else { - xn = wxn = 0; - } - - if (xn || (wxn && (prot_rw & PAGE_WRITE))) { - return prot_rw; - } - return prot_rw | PAGE_EXEC; -} - /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.c b/target/arm/ptw.c index d754273fa1..af9ad42028 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -487,6 +487,134 @@ do_fault: return true; } +/* + * Translate S2 section/page access permissions to protection flags + * @env: CPUARMState + * @s2ap: The 2-bit stage2 access permissions (S2AP) + * @xn: XN (execute-never) bits + * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0 + */ +static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) +{ + int prot = 0; + + if (s2ap & 1) { + prot |= PAGE_READ; + } + if (s2ap & 2) { + prot |= PAGE_WRITE; + } + + if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) { + switch (xn) { + case 0: + prot |= PAGE_EXEC; + break; + case 1: + if (s1_is_el0) { + prot |= PAGE_EXEC; + } + break; + case 2: + break; + case 3: + if (!s1_is_el0) { + prot |= PAGE_EXEC; + } + break; + default: + g_assert_not_reached(); + } + } else { + if (!extract32(xn, 1, 1)) { + if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) { + prot |= PAGE_EXEC; + } + } + } + return prot; +} + +/* + * Translate section/page access permissions to protection flags + * @env: CPUARMState + * @mmu_idx: MMU index indicating required translation regime + * @is_aa64: TRUE if AArch64 + * @ap: The 2-bit simple AP (AP[2:1]) + * @ns: NS (non-secure) bit + * @xn: XN (execute-never) bit + * @pxn: PXN (privileged execute-never) bit + */ +static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, + int ap, int ns, int xn, int pxn) +{ + bool is_user = regime_is_user(env, mmu_idx); + int prot_rw, user_rw; + bool have_wxn; + int wxn = 0; + + assert(mmu_idx != ARMMMUIdx_Stage2); + assert(mmu_idx != ARMMMUIdx_Stage2_S); + + user_rw = simple_ap_to_rw_prot_is_user(ap, true); + if (is_user) { + prot_rw = user_rw; + } else { + if (user_rw && regime_is_pan(env, mmu_idx)) { + /* PAN forbids data accesses but doesn't affect insn fetch */ + prot_rw = 0; + } else { + prot_rw = simple_ap_to_rw_prot_is_user(ap, false); + } + } + + if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) { + return prot_rw; + } + + /* TODO have_wxn should be replaced with + * ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2) + * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE + * compatible processors have EL2, which is required for [U]WXN. + */ + have_wxn = arm_feature(env, ARM_FEATURE_LPAE); + + if (have_wxn) { + wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN; + } + + if (is_aa64) { + if (regime_has_2_ranges(mmu_idx) && !is_user) { + xn = pxn || (user_rw & PAGE_WRITE); + } + } else if (arm_feature(env, ARM_FEATURE_V7)) { + switch (regime_el(env, mmu_idx)) { + case 1: + case 3: + if (is_user) { + xn = xn || !(user_rw & PAGE_READ); + } else { + int uwxn = 0; + if (have_wxn) { + uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN; + } + xn = xn || !(prot_rw & PAGE_READ) || pxn || + (uwxn && (user_rw & PAGE_WRITE)); + } + break; + case 2: + break; + } + } else { + xn = wxn = 0; + } + + if (xn || (wxn && (prot_rw & PAGE_WRITE))) { + return prot_rw; + } + return prot_rw | PAGE_EXEC; +} + /** * get_phys_addr_lpae: perform one stage of page table walk, LPAE format * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index fba650d01c..93147e0b06 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -29,9 +29,6 @@ ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, ARMMMUIdx mmu_idx); bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, int inputsize, int stride, int outputsize); -int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0); -int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, - int ap, int ns, int xn, int pxn); #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From c5168785d208527c45f455619abe2539182a9fff Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 27/55] target/arm: Move check_s2_mmu_setup to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-21-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 70 --------------------------------------------- target/arm/ptw.c | 70 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 2 -- 3 files changed, 70 insertions(+), 72 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 148eb28ba8..2526f4c6c4 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10614,76 +10614,6 @@ int simple_ap_to_rw_prot_is_user(int ap, bool is_user) g_assert_not_reached(); } } - -/* - * check_s2_mmu_setup - * @cpu: ARMCPU - * @is_aa64: True if the translation regime is in AArch64 state - * @startlevel: Suggested starting level - * @inputsize: Bitsize of IPAs - * @stride: Page-table stride (See the ARM ARM) - * - * Returns true if the suggested S2 translation parameters are OK and - * false otherwise. - */ -bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, - int inputsize, int stride, int outputsize) -{ - const int grainsize = stride + 3; - int startsizecheck; - - /* - * Negative levels are usually not allowed... - * Except for FEAT_LPA2, 4k page table, 52-bit address space, which - * begins with level -1. Note that previous feature tests will have - * eliminated this combination if it is not enabled. - */ - if (level < (inputsize == 52 && stride == 9 ? -1 : 0)) { - return false; - } - - startsizecheck = inputsize - ((3 - level) * stride + grainsize); - if (startsizecheck < 1 || startsizecheck > stride + 4) { - return false; - } - - if (is_aa64) { - switch (stride) { - case 13: /* 64KB Pages. */ - if (level == 0 || (level == 1 && outputsize <= 42)) { - return false; - } - break; - case 11: /* 16KB Pages. */ - if (level == 0 || (level == 1 && outputsize <= 40)) { - return false; - } - break; - case 9: /* 4KB Pages. */ - if (level == 0 && outputsize <= 42) { - return false; - } - break; - default: - g_assert_not_reached(); - } - - /* Inputsize checks. */ - if (inputsize > outputsize && - (arm_el_is_aa64(&cpu->env, 1) || inputsize > 40)) { - /* This is CONSTRAINED UNPREDICTABLE and we choose to fault. */ - return false; - } - } else { - /* AArch32 only supports 4KB pages. Assert on that. */ - assert(stride == 9); - - if (level == 0) { - return false; - } - } - return true; -} #endif /* !CONFIG_USER_ONLY */ int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index af9ad42028..525272e99a 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -615,6 +615,76 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } +/* + * check_s2_mmu_setup + * @cpu: ARMCPU + * @is_aa64: True if the translation regime is in AArch64 state + * @startlevel: Suggested starting level + * @inputsize: Bitsize of IPAs + * @stride: Page-table stride (See the ARM ARM) + * + * Returns true if the suggested S2 translation parameters are OK and + * false otherwise. + */ +static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, + int inputsize, int stride, int outputsize) +{ + const int grainsize = stride + 3; + int startsizecheck; + + /* + * Negative levels are usually not allowed... + * Except for FEAT_LPA2, 4k page table, 52-bit address space, which + * begins with level -1. Note that previous feature tests will have + * eliminated this combination if it is not enabled. + */ + if (level < (inputsize == 52 && stride == 9 ? -1 : 0)) { + return false; + } + + startsizecheck = inputsize - ((3 - level) * stride + grainsize); + if (startsizecheck < 1 || startsizecheck > stride + 4) { + return false; + } + + if (is_aa64) { + switch (stride) { + case 13: /* 64KB Pages. */ + if (level == 0 || (level == 1 && outputsize <= 42)) { + return false; + } + break; + case 11: /* 16KB Pages. */ + if (level == 0 || (level == 1 && outputsize <= 40)) { + return false; + } + break; + case 9: /* 4KB Pages. */ + if (level == 0 && outputsize <= 42) { + return false; + } + break; + default: + g_assert_not_reached(); + } + + /* Inputsize checks. */ + if (inputsize > outputsize && + (arm_el_is_aa64(&cpu->env, 1) || inputsize > 40)) { + /* This is CONSTRAINED UNPREDICTABLE and we choose to fault. */ + return false; + } + } else { + /* AArch32 only supports 4KB pages. Assert on that. */ + assert(stride == 9); + + if (level == 0) { + return false; + } + } + return true; +} + /** * get_phys_addr_lpae: perform one stage of page table walk, LPAE format * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 93147e0b06..a71161b01b 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -27,8 +27,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, ARMMMUIdx mmu_idx); -bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, - int inputsize, int stride, int outputsize); #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 2f0ec92e9488e1ac6c7293fe37a5f1c0e1737b36 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 28/55] target/arm: Move aa32_va_parameters to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-22-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 64 --------------------------------------------- target/arm/ptw.c | 64 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 --- 3 files changed, 64 insertions(+), 67 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 2526f4c6c4..f61f1da61e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10771,70 +10771,6 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, } #ifndef CONFIG_USER_ONLY -ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, - ARMMMUIdx mmu_idx) -{ - uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; - uint32_t el = regime_el(env, mmu_idx); - int select, tsz; - bool epd, hpd; - - assert(mmu_idx != ARMMMUIdx_Stage2_S); - - if (mmu_idx == ARMMMUIdx_Stage2) { - /* VTCR */ - bool sext = extract32(tcr, 4, 1); - bool sign = extract32(tcr, 3, 1); - - /* - * If the sign-extend bit is not the same as t0sz[3], the result - * is unpredictable. Flag this as a guest error. - */ - if (sign != sext) { - qemu_log_mask(LOG_GUEST_ERROR, - "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n"); - } - tsz = sextract32(tcr, 0, 4) + 8; - select = 0; - hpd = false; - epd = false; - } else if (el == 2) { - /* HTCR */ - tsz = extract32(tcr, 0, 3); - select = 0; - hpd = extract64(tcr, 24, 1); - epd = false; - } else { - int t0sz = extract32(tcr, 0, 3); - int t1sz = extract32(tcr, 16, 3); - - if (t1sz == 0) { - select = va > (0xffffffffu >> t0sz); - } else { - /* Note that we will detect errors later. */ - select = va >= ~(0xffffffffu >> t1sz); - } - if (!select) { - tsz = t0sz; - epd = extract32(tcr, 7, 1); - hpd = extract64(tcr, 41, 1); - } else { - tsz = t1sz; - epd = extract32(tcr, 23, 1); - hpd = extract64(tcr, 42, 1); - } - /* For aarch32, hpd0 is not enabled without t2e as well. */ - hpd &= extract32(tcr, 6, 1); - } - - return (ARMVAParameters) { - .tsz = tsz, - .select = select, - .epd = epd, - .hpd = hpd, - }; -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 525272e99a..427813ea56 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -615,6 +615,70 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } +static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, + ARMMMUIdx mmu_idx) +{ + uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; + uint32_t el = regime_el(env, mmu_idx); + int select, tsz; + bool epd, hpd; + + assert(mmu_idx != ARMMMUIdx_Stage2_S); + + if (mmu_idx == ARMMMUIdx_Stage2) { + /* VTCR */ + bool sext = extract32(tcr, 4, 1); + bool sign = extract32(tcr, 3, 1); + + /* + * If the sign-extend bit is not the same as t0sz[3], the result + * is unpredictable. Flag this as a guest error. + */ + if (sign != sext) { + qemu_log_mask(LOG_GUEST_ERROR, + "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n"); + } + tsz = sextract32(tcr, 0, 4) + 8; + select = 0; + hpd = false; + epd = false; + } else if (el == 2) { + /* HTCR */ + tsz = extract32(tcr, 0, 3); + select = 0; + hpd = extract64(tcr, 24, 1); + epd = false; + } else { + int t0sz = extract32(tcr, 0, 3); + int t1sz = extract32(tcr, 16, 3); + + if (t1sz == 0) { + select = va > (0xffffffffu >> t0sz); + } else { + /* Note that we will detect errors later. */ + select = va >= ~(0xffffffffu >> t1sz); + } + if (!select) { + tsz = t0sz; + epd = extract32(tcr, 7, 1); + hpd = extract64(tcr, 41, 1); + } else { + tsz = t1sz; + epd = extract32(tcr, 23, 1); + hpd = extract64(tcr, 42, 1); + } + /* For aarch32, hpd0 is not enabled without t2e as well. */ + hpd &= extract32(tcr, 6, 1); + } + + return (ARMVAParameters) { + .tsz = tsz, + .select = select, + .epd = epd, + .hpd = hpd, + }; +} + /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.h b/target/arm/ptw.h index a71161b01b..9314fb4d23 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -25,8 +25,5 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } -ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, - ARMMMUIdx mmu_idx); - #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 4845d3be1239a956a4cedd5c2ca7863837487927 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 29/55] target/arm: Move ap_to_tw_prot etc to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-23-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 77 ------------------------------------------ target/arm/ptw.c | 81 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 10 ------ 3 files changed, 81 insertions(+), 87 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index f61f1da61e..e894afcb49 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10537,83 +10537,6 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) g_assert_not_reached(); } } - -/* Translate section/page access permissions to page - * R/W protection flags - * - * @env: CPUARMState - * @mmu_idx: MMU index indicating required translation regime - * @ap: The 3-bit access permissions (AP[2:0]) - * @domain_prot: The 2-bit domain access permissions - */ -int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot) -{ - bool is_user = regime_is_user(env, mmu_idx); - - if (domain_prot == 3) { - return PAGE_READ | PAGE_WRITE; - } - - switch (ap) { - case 0: - if (arm_feature(env, ARM_FEATURE_V7)) { - return 0; - } - switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) { - case SCTLR_S: - return is_user ? 0 : PAGE_READ; - case SCTLR_R: - return PAGE_READ; - default: - return 0; - } - case 1: - return is_user ? 0 : PAGE_READ | PAGE_WRITE; - case 2: - if (is_user) { - return PAGE_READ; - } else { - return PAGE_READ | PAGE_WRITE; - } - case 3: - return PAGE_READ | PAGE_WRITE; - case 4: /* Reserved. */ - return 0; - case 5: - return is_user ? 0 : PAGE_READ; - case 6: - return PAGE_READ; - case 7: - if (!arm_feature(env, ARM_FEATURE_V6K)) { - return 0; - } - return PAGE_READ; - default: - g_assert_not_reached(); - } -} - -/* Translate section/page access permissions to page - * R/W protection flags. - * - * @ap: The 2-bit simple AP (AP[2:1]) - * @is_user: TRUE if accessing from PL0 - */ -int simple_ap_to_rw_prot_is_user(int ap, bool is_user) -{ - switch (ap) { - case 0: - return is_user ? 0 : PAGE_READ | PAGE_WRITE; - case 1: - return PAGE_READ | PAGE_WRITE; - case 2: - return is_user ? 0 : PAGE_READ; - case 3: - return PAGE_READ; - default: - g_assert_not_reached(); - } -} #endif /* !CONFIG_USER_ONLY */ int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 427813ea56..9ab77c3998 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -211,6 +211,87 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, return true; } +/* + * Translate section/page access permissions to page R/W protection flags + * @env: CPUARMState + * @mmu_idx: MMU index indicating required translation regime + * @ap: The 3-bit access permissions (AP[2:0]) + * @domain_prot: The 2-bit domain access permissions + */ +static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, + int ap, int domain_prot) +{ + bool is_user = regime_is_user(env, mmu_idx); + + if (domain_prot == 3) { + return PAGE_READ | PAGE_WRITE; + } + + switch (ap) { + case 0: + if (arm_feature(env, ARM_FEATURE_V7)) { + return 0; + } + switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) { + case SCTLR_S: + return is_user ? 0 : PAGE_READ; + case SCTLR_R: + return PAGE_READ; + default: + return 0; + } + case 1: + return is_user ? 0 : PAGE_READ | PAGE_WRITE; + case 2: + if (is_user) { + return PAGE_READ; + } else { + return PAGE_READ | PAGE_WRITE; + } + case 3: + return PAGE_READ | PAGE_WRITE; + case 4: /* Reserved. */ + return 0; + case 5: + return is_user ? 0 : PAGE_READ; + case 6: + return PAGE_READ; + case 7: + if (!arm_feature(env, ARM_FEATURE_V6K)) { + return 0; + } + return PAGE_READ; + default: + g_assert_not_reached(); + } +} + +/* + * Translate section/page access permissions to page R/W protection flags. + * @ap: The 2-bit simple AP (AP[2:1]) + * @is_user: TRUE if accessing from PL0 + */ +static int simple_ap_to_rw_prot_is_user(int ap, bool is_user) +{ + switch (ap) { + case 0: + return is_user ? 0 : PAGE_READ | PAGE_WRITE; + case 1: + return PAGE_READ | PAGE_WRITE; + case 2: + return is_user ? 0 : PAGE_READ; + case 3: + return PAGE_READ; + default: + g_assert_not_reached(); + } +} + +static int simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) +{ + return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); +} + static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 9314fb4d23..85ad576794 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -15,15 +15,5 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); -int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, - int ap, int domain_prot); -int simple_ap_to_rw_prot_is_user(int ap, bool is_user); - -static inline int -simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) -{ - return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); -} - #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 0c23d56fc149ada0823648481c09b5d2047570f4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 30/55] target/arm: Move regime_is_user to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-24-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 24 ------------------------ target/arm/ptw.c | 22 ++++++++++++++++++++++ target/arm/ptw.h | 1 - 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index e894afcb49..8deb0fa94c 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10515,30 +10515,6 @@ ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) } #endif /* !CONFIG_USER_ONLY */ -#ifndef CONFIG_USER_ONLY -bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - switch (mmu_idx) { - case ARMMMUIdx_SE10_0: - case ARMMMUIdx_E20_0: - case ARMMMUIdx_SE20_0: - case ARMMMUIdx_Stage1_E0: - case ARMMMUIdx_Stage1_SE0: - case ARMMMUIdx_MUser: - case ARMMMUIdx_MSUser: - case ARMMMUIdx_MUserNegPri: - case ARMMMUIdx_MSUserNegPri: - return true; - default: - return false; - case ARMMMUIdx_E10_0: - case ARMMMUIdx_E10_1: - case ARMMMUIdx_E10_1_PAN: - g_assert_not_reached(); - } -} -#endif /* !CONFIG_USER_ONLY */ - int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 9ab77c3998..8db4b5edf1 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -53,6 +53,28 @@ static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; } +static bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + switch (mmu_idx) { + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_SE20_0: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_SE0: + case ARMMMUIdx_MUser: + case ARMMMUIdx_MSUser: + case ARMMMUIdx_MUserNegPri: + case ARMMMUIdx_MSUserNegPri: + return true; + default: + return false; + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E10_1_PAN: + g_assert_not_reached(); + } +} + static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 85ad576794..3d3061a435 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,7 +11,6 @@ #ifndef CONFIG_USER_ONLY -bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); From 3b318aaeef120a71544771cd84884ccad5fdff0a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 31/55] target/arm: Move regime_ttbr to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-25-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 16 ---------------- target/arm/ptw.c | 16 ++++++++++++++++ target/arm/ptw.h | 1 - 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 8deb0fa94c..fdda87e87e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10475,22 +10475,6 @@ bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; } -/* Return the TTBR associated with this translation regime */ -uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) -{ - if (mmu_idx == ARMMMUIdx_Stage2) { - return env->cp15.vttbr_el2; - } - if (mmu_idx == ARMMMUIdx_Stage2_S) { - return env->cp15.vsttbr_el2; - } - if (ttbrn == 0) { - return env->cp15.ttbr0_el[regime_el(env, mmu_idx)]; - } else { - return env->cp15.ttbr1_el[regime_el(env, mmu_idx)]; - } -} - /* Convert a possible stage1+2 MMU index into the appropriate * stage 1 MMU index */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 8db4b5edf1..dc559e6bdf 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -75,6 +75,22 @@ static bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) } } +/* Return the TTBR associated with this translation regime */ +static uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) +{ + if (mmu_idx == ARMMMUIdx_Stage2) { + return env->cp15.vttbr_el2; + } + if (mmu_idx == ARMMMUIdx_Stage2_S) { + return env->cp15.vsttbr_el2; + } + if (ttbrn == 0) { + return env->cp15.ttbr0_el[regime_el(env, mmu_idx)]; + } else { + return env->cp15.ttbr1_el[regime_el(env, mmu_idx)]; + } +} + static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 3d3061a435..ed152ddaf4 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -12,7 +12,6 @@ #ifndef CONFIG_USER_ONLY bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); -uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 8db1a3a0bba1dbe71a3d363b23416a3c30653ddd Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 32/55] target/arm: Move regime_translation_disabled to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-26-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 47 --------------------------------------------- target/arm/ptw.c | 47 ++++++++++++++++++++++++++++++++++++++++++++- target/arm/ptw.h | 17 ---------------- 3 files changed, 46 insertions(+), 65 deletions(-) delete mode 100644 target/arm/ptw.h diff --git a/target/arm/helper.c b/target/arm/helper.c index fdda87e87e..69b1c060c1 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -36,7 +36,6 @@ #include "semihosting/common-semi.h" #endif #include "cpregs.h" -#include "ptw.h" #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */ @@ -10429,52 +10428,6 @@ uint64_t arm_sctlr(CPUARMState *env, int el) } #ifndef CONFIG_USER_ONLY - -/* Return true if the specified stage of address translation is disabled */ -bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - uint64_t hcr_el2; - - if (arm_feature(env, ARM_FEATURE_M)) { - switch (env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] & - (R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK)) { - case R_V7M_MPU_CTRL_ENABLE_MASK: - /* Enabled, but not for HardFault and NMI */ - return mmu_idx & ARM_MMU_IDX_M_NEGPRI; - case R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK: - /* Enabled for all cases */ - return false; - case 0: - default: - /* HFNMIENA set and ENABLE clear is UNPREDICTABLE, but - * we warned about that in armv7m_nvic.c when the guest set it. - */ - return true; - } - } - - hcr_el2 = arm_hcr_el2_eff(env); - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - /* HCR.DC means HCR.VM behaves as 1 */ - return (hcr_el2 & (HCR_DC | HCR_VM)) == 0; - } - - if (hcr_el2 & HCR_TGE) { - /* TGE means that NS EL0/1 act as if SCTLR_EL1.M is zero */ - if (!regime_is_secure(env, mmu_idx) && regime_el(env, mmu_idx) == 1) { - return true; - } - } - - if ((hcr_el2 & HCR_DC) && arm_mmu_idx_is_stage1_of_2(mmu_idx)) { - /* HCR.DC means SCTLR_EL1.M behaves as 0 */ - return true; - } - - return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; -} - /* Convert a possible stage1+2 MMU index into the appropriate * stage 1 MMU index */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index dc559e6bdf..ec60afd9bf 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -12,7 +12,6 @@ #include "cpu.h" #include "internals.h" #include "idau.h" -#include "ptw.h" static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, @@ -91,6 +90,52 @@ static uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) } } +/* Return true if the specified stage of address translation is disabled */ +static bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + uint64_t hcr_el2; + + if (arm_feature(env, ARM_FEATURE_M)) { + switch (env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] & + (R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK)) { + case R_V7M_MPU_CTRL_ENABLE_MASK: + /* Enabled, but not for HardFault and NMI */ + return mmu_idx & ARM_MMU_IDX_M_NEGPRI; + case R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK: + /* Enabled for all cases */ + return false; + case 0: + default: + /* + * HFNMIENA set and ENABLE clear is UNPREDICTABLE, but + * we warned about that in armv7m_nvic.c when the guest set it. + */ + return true; + } + } + + hcr_el2 = arm_hcr_el2_eff(env); + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + /* HCR.DC means HCR.VM behaves as 1 */ + return (hcr_el2 & (HCR_DC | HCR_VM)) == 0; + } + + if (hcr_el2 & HCR_TGE) { + /* TGE means that NS EL0/1 act as if SCTLR_EL1.M is zero */ + if (!regime_is_secure(env, mmu_idx) && regime_el(env, mmu_idx) == 1) { + return true; + } + } + + if ((hcr_el2 & HCR_DC) && arm_mmu_idx_is_stage1_of_2(mmu_idx)) { + /* HCR.DC means SCTLR_EL1.M behaves as 0 */ + return true; + } + + return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; +} + static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.h b/target/arm/ptw.h deleted file mode 100644 index ed152ddaf4..0000000000 --- a/target/arm/ptw.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * ARM page table walking. - * - * This code is licensed under the GNU GPL v2 or later. - * - * SPDX-License-Identifier: GPL-2.0-or-later - */ - -#ifndef TARGET_ARM_PTW_H -#define TARGET_ARM_PTW_H - -#ifndef CONFIG_USER_ONLY - -bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); - -#endif /* !CONFIG_USER_ONLY */ -#endif /* TARGET_ARM_PTW_H */ From 23971205cf38e63964b7e110df7d5a293277a48e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 33/55] target/arm: Move arm_cpu_get_phys_page_attrs_debug to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-27-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 26 -------------------------- target/arm/ptw.c | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 69b1c060c1..fe1e426f88 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10606,32 +10606,6 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, }; } -#ifndef CONFIG_USER_ONLY -hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, - MemTxAttrs *attrs) -{ - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - hwaddr phys_addr; - target_ulong page_size; - int prot; - bool ret; - ARMMMUFaultInfo fi = {}; - ARMMMUIdx mmu_idx = arm_mmu_idx(env); - ARMCacheAttrs cacheattrs = {}; - - *attrs = (MemTxAttrs) {}; - - ret = get_phys_addr(env, addr, MMU_DATA_LOAD, mmu_idx, &phys_addr, - attrs, &prot, &page_size, &fi, &cacheattrs); - - if (ret) { - return -1; - } - return phys_addr; -} -#endif - /* Note that signed overflow is undefined in C. The following routines are careful to use unsigned types where modulo arithmetic is required. Failure to do so _will_ break on newer gcc. */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index ec60afd9bf..e9f6870d0a 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -2491,3 +2491,27 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, phys_ptr, prot, page_size, fi); } } + +hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, + MemTxAttrs *attrs) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + hwaddr phys_addr; + target_ulong page_size; + int prot; + bool ret; + ARMMMUFaultInfo fi = {}; + ARMMMUIdx mmu_idx = arm_mmu_idx(env); + ARMCacheAttrs cacheattrs = {}; + + *attrs = (MemTxAttrs) {}; + + ret = get_phys_addr(env, addr, MMU_DATA_LOAD, mmu_idx, &phys_addr, + attrs, &prot, &page_size, &fi, &cacheattrs); + + if (ret) { + return -1; + } + return phys_addr; +} From 1d2612553640a79e049708b9d87a9cd3ddb29af6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 34/55] target/arm: Move stage_1_mmu_idx, arm_stage1_mmu_idx to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-28-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 32 -------------------------------- target/arm/ptw.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index fe1e426f88..37cf9fa6ab 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10427,31 +10427,6 @@ uint64_t arm_sctlr(CPUARMState *env, int el) return env->cp15.sctlr_el[el]; } -#ifndef CONFIG_USER_ONLY -/* Convert a possible stage1+2 MMU index into the appropriate - * stage 1 MMU index - */ -ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) -{ - switch (mmu_idx) { - case ARMMMUIdx_SE10_0: - return ARMMMUIdx_Stage1_SE0; - case ARMMMUIdx_SE10_1: - return ARMMMUIdx_Stage1_SE1; - case ARMMMUIdx_SE10_1_PAN: - return ARMMMUIdx_Stage1_SE1_PAN; - case ARMMMUIdx_E10_0: - return ARMMMUIdx_Stage1_E0; - case ARMMMUIdx_E10_1: - return ARMMMUIdx_Stage1_E1; - case ARMMMUIdx_E10_1_PAN: - return ARMMMUIdx_Stage1_E1_PAN; - default: - return mmu_idx; - } -} -#endif /* !CONFIG_USER_ONLY */ - int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { @@ -11081,13 +11056,6 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env) return arm_mmu_idx_el(env, arm_current_el(env)); } -#ifndef CONFIG_USER_ONLY -ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) -{ - return stage_1_mmu_idx(arm_mmu_idx(env)); -} -#endif - static CPUARMTBFlags rebuild_hflags_common(CPUARMState *env, int fp_el, ARMMMUIdx mmu_idx, CPUARMTBFlags flags) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index e9f6870d0a..49e9a1d108 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -47,6 +47,34 @@ unsigned int arm_pamax(ARMCPU *cpu) return pamax_map[parange]; } +/* + * Convert a possible stage1+2 MMU index into the appropriate stage 1 MMU index + */ +ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) +{ + switch (mmu_idx) { + case ARMMMUIdx_SE10_0: + return ARMMMUIdx_Stage1_SE0; + case ARMMMUIdx_SE10_1: + return ARMMMUIdx_Stage1_SE1; + case ARMMMUIdx_SE10_1_PAN: + return ARMMMUIdx_Stage1_SE1_PAN; + case ARMMMUIdx_E10_0: + return ARMMMUIdx_Stage1_E0; + case ARMMMUIdx_E10_1: + return ARMMMUIdx_Stage1_E1; + case ARMMMUIdx_E10_1_PAN: + return ARMMMUIdx_Stage1_E1_PAN; + default: + return mmu_idx; + } +} + +ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) +{ + return stage_1_mmu_idx(arm_mmu_idx(env)); +} + static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) { return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; From 5e79887ba67e22bfd890b72f94c482176a200fbc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 35/55] target/arm: Pass CPUARMState to arm_ld[lq]_ptw The use of ARM_CPU to recover env from cs calls object_class_dynamic_cast, which shows up on the profile. This is pointless, because all callers already have env, and the reverse operation, env_cpu, is only pointer arithmetic. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-29-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/ptw.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 49e9a1d108..4d97a24808 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -241,11 +241,10 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, } /* All loads done in the course of a page table walk go through here. */ -static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, +static uint32_t arm_ldl_ptw(CPUARMState *env, hwaddr addr, bool is_secure, ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; + CPUState *cs = env_cpu(env); MemTxAttrs attrs = {}; MemTxResult result = MEMTX_OK; AddressSpace *as; @@ -270,11 +269,10 @@ static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, +static uint64_t arm_ldq_ptw(CPUARMState *env, hwaddr addr, bool is_secure, ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; + CPUState *cs = env_cpu(env); MemTxAttrs attrs = {}; MemTxResult result = MEMTX_OK; AddressSpace *as; @@ -409,7 +407,6 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, target_ulong *page_size, ARMMMUFaultInfo *fi) { - CPUState *cs = env_cpu(env); int level = 1; uint32_t table; uint32_t desc; @@ -427,7 +424,7 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, fi->type = ARMFault_Translation; goto do_fault; } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -466,7 +463,7 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, /* Fine pagetable. */ table = (desc & 0xfffff000) | ((address >> 8) & 0xffc); } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -531,7 +528,6 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, target_ulong *page_size, ARMMMUFaultInfo *fi) { - CPUState *cs = env_cpu(env); ARMCPU *cpu = env_archcpu(env); int level = 1; uint32_t table; @@ -553,7 +549,7 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, fi->type = ARMFault_Translation; goto do_fault; } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -607,7 +603,7 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, ns = extract32(desc, 3, 1); /* Lookup l2 entry. */ table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -973,7 +969,6 @@ static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); /* Read an LPAE long-descriptor translation table. */ ARMFaultType fault_type = ARMFault_Translation; uint32_t level; @@ -1196,7 +1191,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, descaddr |= (address >> (stride * (4 - level))) & indexmask; descaddr &= ~7ULL; nstable = extract32(tableattrs, 4, 1); - descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi); + descriptor = arm_ldq_ptw(env, descaddr, !nstable, mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; } From f45ce4c35f5e0873bbbc3119eff8539610233b7e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 36/55] target/arm: Rename TBFLAG_A64 ZCR_LEN to VL With SME, the vector length does not only come from ZCR_ELx. Comment that this is either NVL or SVL, like the pseudocode. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-2-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 3 ++- target/arm/helper.c | 2 +- target/arm/translate-a64.c | 2 +- target/arm/translate-a64.h | 2 +- target/arm/translate-sve.c | 2 +- target/arm/translate.h | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 0ee1705a4f..e791ffdd6b 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3241,7 +3241,8 @@ FIELD(TBFLAG_M32, MVE_NO_PRED, 5, 1) /* Not cached. */ */ FIELD(TBFLAG_A64, TBII, 0, 2) FIELD(TBFLAG_A64, SVEEXC_EL, 2, 2) -FIELD(TBFLAG_A64, ZCR_LEN, 4, 4) +/* The current vector length, either NVL or SVL. */ +FIELD(TBFLAG_A64, VL, 4, 4) FIELD(TBFLAG_A64, PAUTH_ACTIVE, 8, 1) FIELD(TBFLAG_A64, BT, 9, 1) FIELD(TBFLAG_A64, BTYPE, 10, 2) /* Not cached. */ diff --git a/target/arm/helper.c b/target/arm/helper.c index 37cf9fa6ab..c228deca75 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11181,7 +11181,7 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, zcr_len = sve_zcr_len_for_el(env, el); } DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); - DP_TBFLAG_A64(flags, ZCR_LEN, zcr_len); + DP_TBFLAG_A64(flags, VL, zcr_len); } sctlr = regime_sctlr(env, stage1); diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 935e1929bb..d438fb89e7 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -14608,7 +14608,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); - dc->sve_len = (EX_TBFLAG_A64(tb_flags, ZCR_LEN) + 1) * 16; + dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); dc->bt = EX_TBFLAG_A64(tb_flags, BT); dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h index f2e8ee0ee1..dbc917ee65 100644 --- a/target/arm/translate-a64.h +++ b/target/arm/translate-a64.h @@ -104,7 +104,7 @@ static inline TCGv_ptr vec_full_reg_ptr(DisasContext *s, int regno) /* Return the byte size of the "whole" vector register, VL / 8. */ static inline int vec_full_reg_size(DisasContext *s) { - return s->sve_len; + return s->vl; } bool disas_sve(DisasContext *, uint32_t); diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index 836511d719..67761bf2cc 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -111,7 +111,7 @@ static inline int pred_full_reg_offset(DisasContext *s, int regno) /* Return the byte size of the whole predicate register, VL / 64. */ static inline int pred_full_reg_size(DisasContext *s) { - return s->sve_len >> 3; + return s->vl >> 3; } /* Round up the size of a register to a size allowed by diff --git a/target/arm/translate.h b/target/arm/translate.h index 9f0bb270c5..f473a21ed4 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -42,7 +42,7 @@ typedef struct DisasContext { bool ns; /* Use non-secure CPREG bank on access */ int fp_excp_el; /* FP exception EL or 0 if enabled */ int sve_excp_el; /* SVE exception EL or 0 if enabled */ - int sve_len; /* SVE vector length in bytes */ + int vl; /* current vector length in bytes */ /* Flag indicating that exceptions from secure mode are routed to EL3. */ bool secure_routed_to_el3; bool vfp_enabled; /* FP enabled via FPSCR.EN */ From 8b599e5c0274cd5e4a33be6a10d6cd04631a74d1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 37/55] linux-user/aarch64: Introduce sve_vq Add an interface function to extract the digested vector length rather than the raw zcr_el[1] value. This fixes an incorrect return from do_prctl_set_vl where we didn't take into account the set of vector lengths supported by the cpu. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-3-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- linux-user/aarch64/signal.c | 4 ++-- linux-user/aarch64/target_prctl.h | 20 +++++++++++++------- target/arm/cpu.h | 11 +++++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/linux-user/aarch64/signal.c b/linux-user/aarch64/signal.c index 7de4c96eb9..7da0e36c6d 100644 --- a/linux-user/aarch64/signal.c +++ b/linux-user/aarch64/signal.c @@ -315,7 +315,7 @@ static int target_restore_sigframe(CPUARMState *env, case TARGET_SVE_MAGIC: if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { - vq = (env->vfp.zcr_el[1] & 0xf) + 1; + vq = sve_vq(env); sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(vq), 16); if (!sve && size == sve_size) { sve = (struct target_sve_context *)ctx; @@ -434,7 +434,7 @@ static void target_setup_frame(int usig, struct target_sigaction *ka, /* SVE state needs saving only if it exists. */ if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { - vq = (env->vfp.zcr_el[1] & 0xf) + 1; + vq = sve_vq(env); sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(vq), 16); sve_ofs = alloc_sigframe_space(sve_size, &layout); } diff --git a/linux-user/aarch64/target_prctl.h b/linux-user/aarch64/target_prctl.h index 3f5a5d3933..1d440ffbea 100644 --- a/linux-user/aarch64/target_prctl.h +++ b/linux-user/aarch64/target_prctl.h @@ -10,7 +10,7 @@ static abi_long do_prctl_get_vl(CPUArchState *env) { ARMCPU *cpu = env_archcpu(env); if (cpu_isar_feature(aa64_sve, cpu)) { - return ((cpu->env.vfp.zcr_el[1] & 0xf) + 1) * 16; + return sve_vq(env) * 16; } return -TARGET_EINVAL; } @@ -25,18 +25,24 @@ static abi_long do_prctl_set_vl(CPUArchState *env, abi_long arg2) */ if (cpu_isar_feature(aa64_sve, env_archcpu(env)) && arg2 >= 0 && arg2 <= 512 * 16 && !(arg2 & 15)) { - ARMCPU *cpu = env_archcpu(env); uint32_t vq, old_vq; - old_vq = (env->vfp.zcr_el[1] & 0xf) + 1; - vq = MAX(arg2 / 16, 1); - vq = MIN(vq, cpu->sve_max_vq); + old_vq = sve_vq(env); + /* + * Bound the value of arg2, so that we know that it fits into + * the 4-bit field in ZCR_EL1. Rely on the hflags rebuild to + * sort out the length supported by the cpu. + */ + vq = MAX(arg2 / 16, 1); + vq = MIN(vq, ARM_MAX_VQ); + env->vfp.zcr_el[1] = vq - 1; + arm_rebuild_hflags(env); + + vq = sve_vq(env); if (vq < old_vq) { aarch64_sve_narrow_vq(env, vq); } - env->vfp.zcr_el[1] = vq - 1; - arm_rebuild_hflags(env); return vq * 16; } return -TARGET_EINVAL; diff --git a/target/arm/cpu.h b/target/arm/cpu.h index e791ffdd6b..f5af88b686 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3286,6 +3286,17 @@ static inline int cpu_mmu_index(CPUARMState *env, bool ifetch) return EX_TBFLAG_ANY(env->hflags, MMUIDX); } +/** + * sve_vq + * @env: the cpu context + * + * Return the VL cached within env->hflags, in units of quadwords. + */ +static inline int sve_vq(CPUARMState *env) +{ + return EX_TBFLAG_A64(env->hflags, VL) + 1; +} + static inline bool bswap_code(bool sctlr_b) { #ifdef CONFIG_USER_ONLY From 61a8c23a3b56607ee6e745b4cf7975170df88801 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 38/55] target/arm: Remove route_to_el2 check from sve_exception_el We handle this routing in raise_exception. Promoting the value early means that we can't directly compare FPEXC_EL and SVEEXC_EL. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-4-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index c228deca75..1bd77af7e5 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6149,8 +6149,7 @@ int sve_exception_el(CPUARMState *env, int el) /* fall through */ case 0: case 2: - /* route_to_el2 */ - return hcr_el2 & HCR_TGE ? 2 : 1; + return 1; } /* Check CPACR.FPEN. */ From 397d922c6248509c6d490f82088f00cbc716287c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 39/55] target/arm: Remove fp checks from sve_exception_el Instead of checking these bits in fp_exception_el and also in sve_exception_el, document that we must compare the results. The only place where we have not already checked that FP EL is zero is in rebuild_hflags_a64. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-5-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 58 +++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 1bd77af7e5..4f4044c688 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6129,11 +6129,15 @@ static const ARMCPRegInfo minimal_ras_reginfo[] = { .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.vsesr_el2) }, }; -/* Return the exception level to which exceptions should be taken - * via SVEAccessTrap. If an exception should be routed through - * AArch64.AdvSIMDFPAccessTrap, return 0; fp_exception_el should - * take care of raising that exception. - * C.f. the ARM pseudocode function CheckSVEEnabled. +/* + * Return the exception level to which exceptions should be taken + * via SVEAccessTrap. This excludes the check for whether the exception + * should be routed through AArch64.AdvSIMDFPAccessTrap. That can easily + * be found by testing 0 < fp_exception_el < sve_exception_el. + * + * C.f. the ARM pseudocode function CheckSVEEnabled. Note that the + * pseudocode does *not* separate out the FP trap checks, but has them + * all in one function. */ int sve_exception_el(CPUARMState *env, int el) { @@ -6151,18 +6155,6 @@ int sve_exception_el(CPUARMState *env, int el) case 2: return 1; } - - /* Check CPACR.FPEN. */ - switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, FPEN)) { - case 1: - if (el != 0) { - break; - } - /* fall through */ - case 0: - case 2: - return 0; - } } /* @@ -6180,24 +6172,10 @@ int sve_exception_el(CPUARMState *env, int el) case 2: return 2; } - - switch (FIELD_EX32(env->cp15.cptr_el[2], CPTR_EL2, FPEN)) { - case 1: - if (el == 2 || !(hcr_el2 & HCR_TGE)) { - break; - } - /* fall through */ - case 0: - case 2: - return 0; - } } else if (arm_is_el2_enabled(env)) { if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TZ)) { return 2; } - if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TFP)) { - return 0; - } } } @@ -11168,19 +11146,21 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { int sve_el = sve_exception_el(env, el); - uint32_t zcr_len; /* - * If SVE is disabled, but FP is enabled, - * then the effective len is 0. + * If either FP or SVE are disabled, translator does not need len. + * If SVE EL > FP EL, FP exception has precedence, and translator + * does not need SVE EL. Save potential re-translations by forcing + * the unneeded data to zero. */ - if (sve_el != 0 && fp_el == 0) { - zcr_len = 0; - } else { - zcr_len = sve_zcr_len_for_el(env, el); + if (fp_el != 0) { + if (sve_el > fp_el) { + sve_el = 0; + } + } else if (sve_el == 0) { + DP_TBFLAG_A64(flags, VL, sve_zcr_len_for_el(env, el)); } DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); - DP_TBFLAG_A64(flags, VL, zcr_len); } sctlr = regime_sctlr(env, stage1); From 19668718ad188e1b6a162bb52357a67fd407c96c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 40/55] target/arm: Add el_is_in_host This (newish) ARM pseudocode function is easier to work with than open-coded tests for HCR_E2H etc. Use of the function will be staged into the code base in parts. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-6-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 28 ++++++++++++++++++++++++++++ target/arm/internals.h | 2 ++ 2 files changed, 30 insertions(+) diff --git a/target/arm/helper.c b/target/arm/helper.c index 4f4044c688..322508170e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -5282,6 +5282,34 @@ uint64_t arm_hcr_el2_eff(CPUARMState *env) return ret; } +/* + * Corresponds to ARM pseudocode function ELIsInHost(). + */ +bool el_is_in_host(CPUARMState *env, int el) +{ + uint64_t mask; + + /* + * Since we only care about E2H and TGE, we can skip arm_hcr_el2_eff(). + * Perform the simplest bit tests first, and validate EL2 afterward. + */ + if (el & 1) { + return false; /* EL1 or EL3 */ + } + + /* + * Note that hcr_write() checks isar_feature_aa64_vh(), + * aka HaveVirtHostExt(), in allowing HCR_E2H to be set. + */ + mask = el ? HCR_E2H : HCR_E2H | HCR_TGE; + if ((env->cp15.hcr_el2 & mask) != mask) { + return false; + } + + /* TGE and/or E2H set: double check those bits are currently legal. */ + return arm_is_el2_enabled(env) && arm_el_is_aa64(env, 2); +} + static void hcrx_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { diff --git a/target/arm/internals.h b/target/arm/internals.h index 1d83146d56..ceaddcbfd6 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1347,6 +1347,8 @@ static inline void define_cortex_a72_a57_a53_cp_reginfo(ARMCPU *cpu) { } void define_cortex_a72_a57_a53_cp_reginfo(ARMCPU *cpu); #endif +bool el_is_in_host(CPUARMState *env, int el); + void aa32_max_features(ARMCPU *cpu); #endif From c6225bebc2498e9dd85bdd4de1840925650c7a9f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 41/55] target/arm: Use el_is_in_host for sve_zcr_len_for_el The ARM pseudocode function NVL uses this predicate now, and I think it's a bit clearer. Simplify the pseudocode condition by noting that IsInHost is always false for EL1. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-7-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 322508170e..6b17c0f507 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6238,8 +6238,7 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) ARMCPU *cpu = env_archcpu(env); uint32_t zcr_len = cpu->sve_max_vq - 1; - if (el <= 1 && - (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + if (el <= 1 && !el_is_in_host(env, el)) { zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[1]); } if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) { From aa4451b60e6a2b1bc4e9a8f6bbf90b266e2a4390 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 42/55] target/arm: Use el_is_in_host for sve_exception_el The ARM pseudocode function CheckNormalSVEEnabled uses this predicate now, and I think it's a bit clearer. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-8-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 6b17c0f507..40b60b1eea 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6170,9 +6170,7 @@ static const ARMCPRegInfo minimal_ras_reginfo[] = { int sve_exception_el(CPUARMState *env, int el) { #ifndef CONFIG_USER_ONLY - uint64_t hcr_el2 = arm_hcr_el2_eff(env); - - if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + if (el <= 1 && !el_is_in_host(env, el)) { switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, ZEN)) { case 1: if (el != 0) { @@ -6189,6 +6187,7 @@ int sve_exception_el(CPUARMState *env, int el) * CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */ if (el <= 2) { + uint64_t hcr_el2 = arm_hcr_el2_eff(env); if (hcr_el2 & HCR_E2H) { switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, ZEN)) { case 1: From 7d38cb92aad848b9c935588815d0b3a5e157aabb Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 43/55] target/arm: Hoist arm_is_el2_enabled check in sve_exception_el This check is buried within arm_hcr_el2_eff(), but since we have to have the explicit check for CPTR_EL2.TZ, we might as well just check it once at the beginning of the block. Once this is done, we can test HCR_EL2.{E2H,TGE} directly, rather than going through arm_hcr_el2_eff(). Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-9-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 40b60b1eea..61e8026d0e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6183,15 +6183,12 @@ int sve_exception_el(CPUARMState *env, int el) } } - /* - * CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). - */ - if (el <= 2) { - uint64_t hcr_el2 = arm_hcr_el2_eff(env); - if (hcr_el2 & HCR_E2H) { + if (el <= 2 && arm_is_el2_enabled(env)) { + /* CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */ + if (env->cp15.hcr_el2 & HCR_E2H) { switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, ZEN)) { case 1: - if (el != 0 || !(hcr_el2 & HCR_TGE)) { + if (el != 0 || !(env->cp15.hcr_el2 & HCR_TGE)) { break; } /* fall through */ @@ -6199,7 +6196,7 @@ int sve_exception_el(CPUARMState *env, int el) case 2: return 2; } - } else if (arm_is_el2_enabled(env)) { + } else { if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TZ)) { return 2; } From 87252bdecdafd49f8f4da6244a40415c27053bc3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 44/55] target/arm: Do not use aarch64_sve_zcr_get_valid_len in reset We don't need to constrain the value set in zcr_el[1], because it will be done by sve_zcr_len_for_el. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-10-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index d2bd74c2ed..0621944167 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -208,8 +208,7 @@ static void arm_cpu_reset(DeviceState *dev) CPACR_EL1, ZEN, 3); /* with reasonable vector length */ if (cpu_isar_feature(aa64_sve, cpu)) { - env->vfp.zcr_el[1] = - aarch64_sve_zcr_get_valid_len(cpu, cpu->sve_default_vq - 1); + env->vfp.zcr_el[1] = cpu->sve_default_vq - 1; } /* * Enable 48-bit address space (TODO: take reserved_va into account). From 9b5f422559a528f4117aa9c3098fddf3d7c535ba Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 45/55] target/arm: Merge aarch64_sve_zcr_get_valid_len into caller This function is used only once, and will need modification for Streaming SVE mode. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-11-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 30 +++++++++++------------------- target/arm/internals.h | 11 ----------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 61e8026d0e..de159c644c 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6212,39 +6212,31 @@ int sve_exception_el(CPUARMState *env, int el) return 0; } -uint32_t aarch64_sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len) -{ - uint32_t end_len; - - start_len = MIN(start_len, ARM_MAX_VQ - 1); - end_len = start_len; - - if (!test_bit(start_len, cpu->sve_vq_map)) { - end_len = find_last_bit(cpu->sve_vq_map, start_len); - assert(end_len < start_len); - } - return end_len; -} - /* * Given that SVE is enabled, return the vector length for EL. */ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) { ARMCPU *cpu = env_archcpu(env); - uint32_t zcr_len = cpu->sve_max_vq - 1; + uint32_t len = cpu->sve_max_vq - 1; + uint32_t end_len; if (el <= 1 && !el_is_in_host(env, el)) { - zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[1]); + len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[1]); } if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) { - zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[2]); + len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[2]); } if (arm_feature(env, ARM_FEATURE_EL3)) { - zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[3]); + len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[3]); } - return aarch64_sve_zcr_get_valid_len(cpu, zcr_len); + end_len = len; + if (!test_bit(len, cpu->sve_vq_map)) { + end_len = find_last_bit(cpu->sve_vq_map, len); + assert(end_len < len); + } + return end_len; } static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, diff --git a/target/arm/internals.h b/target/arm/internals.h index ceaddcbfd6..79eb463753 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -189,17 +189,6 @@ void arm_translate_init(void); void arm_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb); #endif /* CONFIG_TCG */ -/** - * aarch64_sve_zcr_get_valid_len: - * @cpu: cpu context - * @start_len: maximum len to consider - * - * Return the maximum supported sve vector length <= @start_len. - * Note that both @start_len and the return value are in units - * of ZCR_ELx.LEN, so the vector bit length is (x + 1) * 128. - */ -uint32_t aarch64_sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len); - enum arm_fprounding { FPROUNDING_TIEEVEN, FPROUNDING_POSINF, From 886902ece71b5e795fea3e052a32f047d2f8fe33 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 46/55] target/arm: Use uint32_t instead of bitmap for sve vq's The bitmap need only hold 15 bits; bitmap is over-complicated. We can simplify operations quite a bit with plain logical ops. The introduction of SVE_VQ_POW2_MAP eliminates the need for looping in order to search for powers of two. Simply perform the logical ops and use count leading or trailing zeros as required to find the result. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-12-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 6 +-- target/arm/cpu64.c | 117 ++++++++++++++++++++--------------------- target/arm/helper.c | 9 +--- target/arm/internals.h | 5 ++ target/arm/kvm64.c | 36 +++---------- target/arm/kvm_arm.h | 7 ++- 6 files changed, 75 insertions(+), 105 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index f5af88b686..73f24a5760 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -1041,9 +1041,9 @@ struct ArchCPU { * Bits set in sve_vq_supported represent valid vector lengths for * the CPU type. */ - DECLARE_BITMAP(sve_vq_map, ARM_MAX_VQ); - DECLARE_BITMAP(sve_vq_init, ARM_MAX_VQ); - DECLARE_BITMAP(sve_vq_supported, ARM_MAX_VQ); + uint32_t sve_vq_map; + uint32_t sve_vq_init; + uint32_t sve_vq_supported; /* Generic timer counter frequency, in Hz */ uint64_t gt_cntfrq_hz; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index cce68dd82a..15665c962b 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -355,8 +355,11 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * any of the above. Finally, if SVE is not disabled, then at least one * vector length must be enabled. */ - DECLARE_BITMAP(tmp, ARM_MAX_VQ); - uint32_t vq, max_vq = 0; + uint32_t vq_map = cpu->sve_vq_map; + uint32_t vq_init = cpu->sve_vq_init; + uint32_t vq_supported; + uint32_t vq_mask = 0; + uint32_t tmp, vq, max_vq = 0; /* * CPU models specify a set of supported vector lengths which are @@ -364,10 +367,16 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * in the supported bitmap results in an error. When KVM is enabled we * fetch the supported bitmap from the host. */ - if (kvm_enabled() && kvm_arm_sve_supported()) { - kvm_arm_sve_get_vls(CPU(cpu), cpu->sve_vq_supported); - } else if (kvm_enabled()) { - assert(!cpu_isar_feature(aa64_sve, cpu)); + if (kvm_enabled()) { + if (kvm_arm_sve_supported()) { + cpu->sve_vq_supported = kvm_arm_sve_get_vls(CPU(cpu)); + vq_supported = cpu->sve_vq_supported; + } else { + assert(!cpu_isar_feature(aa64_sve, cpu)); + vq_supported = 0; + } + } else { + vq_supported = cpu->sve_vq_supported; } /* @@ -375,8 +384,9 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * From the properties, sve_vq_map implies sve_vq_init. * Check first for any sve enabled. */ - if (!bitmap_empty(cpu->sve_vq_map, ARM_MAX_VQ)) { - max_vq = find_last_bit(cpu->sve_vq_map, ARM_MAX_VQ) + 1; + if (vq_map != 0) { + max_vq = 32 - clz32(vq_map); + vq_mask = MAKE_64BIT_MASK(0, max_vq); if (cpu->sve_max_vq && max_vq > cpu->sve_max_vq) { error_setg(errp, "cannot enable sve%d", max_vq * 128); @@ -392,15 +402,10 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * For KVM we have to automatically enable all supported unitialized * lengths, even when the smaller lengths are not all powers-of-two. */ - bitmap_andnot(tmp, cpu->sve_vq_supported, cpu->sve_vq_init, max_vq); - bitmap_or(cpu->sve_vq_map, cpu->sve_vq_map, tmp, max_vq); + vq_map |= vq_supported & ~vq_init & vq_mask; } else { /* Propagate enabled bits down through required powers-of-two. */ - for (vq = pow2floor(max_vq); vq >= 1; vq >>= 1) { - if (!test_bit(vq - 1, cpu->sve_vq_init)) { - set_bit(vq - 1, cpu->sve_vq_map); - } - } + vq_map |= SVE_VQ_POW2_MAP & ~vq_init & vq_mask; } } else if (cpu->sve_max_vq == 0) { /* @@ -413,25 +418,18 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) if (kvm_enabled()) { /* Disabling a supported length disables all larger lengths. */ - for (vq = 1; vq <= ARM_MAX_VQ; ++vq) { - if (test_bit(vq - 1, cpu->sve_vq_init) && - test_bit(vq - 1, cpu->sve_vq_supported)) { - break; - } - } + tmp = vq_init & vq_supported; } else { /* Disabling a power-of-two disables all larger lengths. */ - for (vq = 1; vq <= ARM_MAX_VQ; vq <<= 1) { - if (test_bit(vq - 1, cpu->sve_vq_init)) { - break; - } - } + tmp = vq_init & SVE_VQ_POW2_MAP; } + vq = ctz32(tmp) + 1; max_vq = vq <= ARM_MAX_VQ ? vq - 1 : ARM_MAX_VQ; - bitmap_andnot(cpu->sve_vq_map, cpu->sve_vq_supported, - cpu->sve_vq_init, max_vq); - if (max_vq == 0 || bitmap_empty(cpu->sve_vq_map, max_vq)) { + vq_mask = MAKE_64BIT_MASK(0, max_vq); + vq_map = vq_supported & ~vq_init & vq_mask; + + if (max_vq == 0 || vq_map == 0) { error_setg(errp, "cannot disable sve%d", vq * 128); error_append_hint(errp, "Disabling sve%d results in all " "vector lengths being disabled.\n", @@ -441,7 +439,8 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) return; } - max_vq = find_last_bit(cpu->sve_vq_map, max_vq) + 1; + max_vq = 32 - clz32(vq_map); + vq_mask = MAKE_64BIT_MASK(0, max_vq); } /* @@ -451,9 +450,9 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) */ if (cpu->sve_max_vq != 0) { max_vq = cpu->sve_max_vq; + vq_mask = MAKE_64BIT_MASK(0, max_vq); - if (!test_bit(max_vq - 1, cpu->sve_vq_map) && - test_bit(max_vq - 1, cpu->sve_vq_init)) { + if (vq_init & ~vq_map & (1 << (max_vq - 1))) { error_setg(errp, "cannot disable sve%d", max_vq * 128); error_append_hint(errp, "The maximum vector length must be " "enabled, sve-max-vq=%d (%d bits)\n", @@ -462,8 +461,7 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) } /* Set all bits not explicitly set within sve-max-vq. */ - bitmap_complement(tmp, cpu->sve_vq_init, max_vq); - bitmap_or(cpu->sve_vq_map, cpu->sve_vq_map, tmp, max_vq); + vq_map |= ~vq_init & vq_mask; } /* @@ -472,13 +470,14 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * are clear, just in case anybody looks. */ assert(max_vq != 0); - bitmap_clear(cpu->sve_vq_map, max_vq, ARM_MAX_VQ - max_vq); + assert(vq_mask != 0); + vq_map &= vq_mask; /* Ensure the set of lengths matches what is supported. */ - bitmap_xor(tmp, cpu->sve_vq_map, cpu->sve_vq_supported, max_vq); - if (!bitmap_empty(tmp, max_vq)) { - vq = find_last_bit(tmp, max_vq) + 1; - if (test_bit(vq - 1, cpu->sve_vq_map)) { + tmp = vq_map ^ (vq_supported & vq_mask); + if (tmp) { + vq = 32 - clz32(tmp); + if (vq_map & (1 << (vq - 1))) { if (cpu->sve_max_vq) { error_setg(errp, "cannot set sve-max-vq=%d", cpu->sve_max_vq); error_append_hint(errp, "This CPU does not support " @@ -502,15 +501,15 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) return; } else { /* Ensure all required powers-of-two are enabled. */ - for (vq = pow2floor(max_vq); vq >= 1; vq >>= 1) { - if (!test_bit(vq - 1, cpu->sve_vq_map)) { - error_setg(errp, "cannot disable sve%d", vq * 128); - error_append_hint(errp, "sve%d is required as it " - "is a power-of-two length smaller " - "than the maximum, sve%d\n", - vq * 128, max_vq * 128); - return; - } + tmp = SVE_VQ_POW2_MAP & vq_mask & ~vq_map; + if (tmp) { + vq = 32 - clz32(tmp); + error_setg(errp, "cannot disable sve%d", vq * 128); + error_append_hint(errp, "sve%d is required as it " + "is a power-of-two length smaller " + "than the maximum, sve%d\n", + vq * 128, max_vq * 128); + return; } } } @@ -530,6 +529,7 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) /* From now on sve_max_vq is the actual maximum supported length. */ cpu->sve_max_vq = max_vq; + cpu->sve_vq_map = vq_map; } static void cpu_max_get_sve_max_vq(Object *obj, Visitor *v, const char *name, @@ -590,7 +590,7 @@ static void cpu_arm_get_sve_vq(Object *obj, Visitor *v, const char *name, if (!cpu_isar_feature(aa64_sve, cpu)) { value = false; } else { - value = test_bit(vq - 1, cpu->sve_vq_map); + value = extract32(cpu->sve_vq_map, vq - 1, 1); } visit_type_bool(v, name, &value, errp); } @@ -612,12 +612,8 @@ static void cpu_arm_set_sve_vq(Object *obj, Visitor *v, const char *name, return; } - if (value) { - set_bit(vq - 1, cpu->sve_vq_map); - } else { - clear_bit(vq - 1, cpu->sve_vq_map); - } - set_bit(vq - 1, cpu->sve_vq_init); + cpu->sve_vq_map = deposit32(cpu->sve_vq_map, vq - 1, 1, value); + cpu->sve_vq_init |= 1 << (vq - 1); } static bool cpu_arm_get_sve(Object *obj, Error **errp) @@ -979,7 +975,7 @@ static void aarch64_max_initfn(Object *obj) cpu->dcz_blocksize = 7; /* 512 bytes */ #endif - bitmap_fill(cpu->sve_vq_supported, ARM_MAX_VQ); + cpu->sve_vq_supported = MAKE_64BIT_MASK(0, ARM_MAX_VQ); aarch64_add_pauth_properties(obj); aarch64_add_sve_properties(obj); @@ -1026,12 +1022,11 @@ static void aarch64_a64fx_initfn(Object *obj) cpu->gic_vprebits = 5; cpu->gic_pribits = 5; - /* Suppport of A64FX's vector length are 128,256 and 512bit only */ + /* The A64FX supports only 128, 256 and 512 bit vector lengths */ aarch64_add_sve_properties(obj); - bitmap_zero(cpu->sve_vq_supported, ARM_MAX_VQ); - set_bit(0, cpu->sve_vq_supported); /* 128bit */ - set_bit(1, cpu->sve_vq_supported); /* 256bit */ - set_bit(3, cpu->sve_vq_supported); /* 512bit */ + cpu->sve_vq_supported = (1 << 0) /* 128bit */ + | (1 << 1) /* 256bit */ + | (1 << 3); /* 512bit */ cpu->isar.reset_pmcr_el0 = 0x46014040; diff --git a/target/arm/helper.c b/target/arm/helper.c index de159c644c..90aac6bc12 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6219,7 +6219,6 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) { ARMCPU *cpu = env_archcpu(env); uint32_t len = cpu->sve_max_vq - 1; - uint32_t end_len; if (el <= 1 && !el_is_in_host(env, el)) { len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[1]); @@ -6231,12 +6230,8 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[3]); } - end_len = len; - if (!test_bit(len, cpu->sve_vq_map)) { - end_len = find_last_bit(cpu->sve_vq_map, len); - assert(end_len < len); - } - return end_len; + len = 31 - clz32(cpu->sve_vq_map & MAKE_64BIT_MASK(0, len + 1)); + return len; } static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, diff --git a/target/arm/internals.h b/target/arm/internals.h index 79eb463753..a1bae4588a 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1340,4 +1340,9 @@ bool el_is_in_host(CPUARMState *env, int el); void aa32_max_features(ARMCPU *cpu); +/* Powers of 2 for sve_vq_map et al. */ +#define SVE_VQ_POW2_MAP \ + ((1 << (1 - 1)) | (1 << (2 - 1)) | \ + (1 << (4 - 1)) | (1 << (8 - 1)) | (1 << (16 - 1))) + #endif diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index 363032da90..b3f635fc95 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -760,15 +760,13 @@ bool kvm_arm_steal_time_supported(void) QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); -void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) +uint32_t kvm_arm_sve_get_vls(CPUState *cs) { /* Only call this function if kvm_arm_sve_supported() returns true. */ static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS]; static bool probed; uint32_t vq = 0; - int i, j; - - bitmap_zero(map, ARM_MAX_VQ); + int i; /* * KVM ensures all host CPUs support the same set of vector lengths. @@ -809,46 +807,24 @@ void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) if (vq > ARM_MAX_VQ) { warn_report("KVM supports vector lengths larger than " "QEMU can enable"); + vls[0] &= MAKE_64BIT_MASK(0, ARM_MAX_VQ); } } - for (i = 0; i < KVM_ARM64_SVE_VLS_WORDS; ++i) { - if (!vls[i]) { - continue; - } - for (j = 1; j <= 64; ++j) { - vq = j + i * 64; - if (vq > ARM_MAX_VQ) { - return; - } - if (vls[i] & (1UL << (j - 1))) { - set_bit(vq - 1, map); - } - } - } + return vls[0]; } static int kvm_arm_sve_set_vls(CPUState *cs) { - uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = {0}; + ARMCPU *cpu = ARM_CPU(cs); + uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq_map }; struct kvm_one_reg reg = { .id = KVM_REG_ARM64_SVE_VLS, .addr = (uint64_t)&vls[0], }; - ARMCPU *cpu = ARM_CPU(cs); - uint32_t vq; - int i, j; assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX); - for (vq = 1; vq <= cpu->sve_max_vq; ++vq) { - if (test_bit(vq - 1, cpu->sve_vq_map)) { - i = (vq - 1) / 64; - j = (vq - 1) % 64; - vls[i] |= 1UL << j; - } - } - return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); } diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index b7f78b5215..99017b635c 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -239,13 +239,12 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf); /** * kvm_arm_sve_get_vls: * @cs: CPUState - * @map: bitmap to fill in * * Get all the SVE vector lengths supported by the KVM host, setting * the bits corresponding to their length in quadwords minus one - * (vq - 1) in @map up to ARM_MAX_VQ. + * (vq - 1) up to ARM_MAX_VQ. Return the resulting map. */ -void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map); +uint32_t kvm_arm_sve_get_vls(CPUState *cs); /** * kvm_arm_set_cpu_features_from_host: @@ -439,7 +438,7 @@ static inline void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) g_assert_not_reached(); } -static inline void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) +static inline uint32_t kvm_arm_sve_get_vls(CPUState *cs) { g_assert_not_reached(); } From 5ef3cc563699d9b96e6b1acb15c7c02cf75d8266 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 47/55] target/arm: Rename sve_zcr_len_for_el to sve_vqm1_for_el This will be used for both Normal and Streaming SVE, and the value does not necessarily come from ZCR_ELx. While we're at it, emphasize the units in which the value is returned. Patch produced by git grep -l sve_zcr_len_for_el | \ xargs -n1 sed -i 's/sve_zcr_len_for_el/sve_vqm1_for_el/g' and then adding a function comment. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-13-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/arch_dump.c | 2 +- target/arm/cpu.c | 2 +- target/arm/cpu.h | 11 ++++++++++- target/arm/gdbstub64.c | 2 +- target/arm/helper.c | 12 ++++++------ 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/target/arm/arch_dump.c b/target/arm/arch_dump.c index 0184845310..b1f040e69f 100644 --- a/target/arm/arch_dump.c +++ b/target/arm/arch_dump.c @@ -166,7 +166,7 @@ static off_t sve_fpcr_offset(uint32_t vq) static uint32_t sve_current_vq(CPUARMState *env) { - return sve_zcr_len_for_el(env, arm_current_el(env)) + 1; + return sve_vqm1_for_el(env, arm_current_el(env)) + 1; } static size_t sve_size_vq(uint32_t vq) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index 0621944167..1b5d535788 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -925,7 +925,7 @@ static void aarch64_cpu_dump_state(CPUState *cs, FILE *f, int flags) vfp_get_fpcr(env), vfp_get_fpsr(env)); if (cpu_isar_feature(aa64_sve, cpu) && sve_exception_el(env, el) == 0) { - int j, zcr_len = sve_zcr_len_for_el(env, el); + int j, zcr_len = sve_vqm1_for_el(env, el); for (i = 0; i <= FFR_PRED_NUM; i++) { bool eol; diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 73f24a5760..e45b5cb7fe 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -1132,7 +1132,16 @@ void aarch64_sync_64_to_32(CPUARMState *env); int fp_exception_el(CPUARMState *env, int cur_el); int sve_exception_el(CPUARMState *env, int cur_el); -uint32_t sve_zcr_len_for_el(CPUARMState *env, int el); + +/** + * sve_vqm1_for_el: + * @env: CPUARMState + * @el: exception level + * + * Compute the current SVE vector length for @el, in units of + * Quadwords Minus 1 -- the same scale used for ZCR_ELx.LEN. + */ +uint32_t sve_vqm1_for_el(CPUARMState *env, int el); static inline bool is_a64(CPUARMState *env) { diff --git a/target/arm/gdbstub64.c b/target/arm/gdbstub64.c index 596878666d..07a6746944 100644 --- a/target/arm/gdbstub64.c +++ b/target/arm/gdbstub64.c @@ -152,7 +152,7 @@ int arm_gdb_get_svereg(CPUARMState *env, GByteArray *buf, int reg) * We report in Vector Granules (VG) which is 64bit in a Z reg * while the ZCR works in Vector Quads (VQ) which is 128bit chunks. */ - int vq = sve_zcr_len_for_el(env, arm_current_el(env)) + 1; + int vq = sve_vqm1_for_el(env, arm_current_el(env)) + 1; return gdb_get_reg64(buf, vq * 2); } default: diff --git a/target/arm/helper.c b/target/arm/helper.c index 90aac6bc12..400f7cd1db 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6215,7 +6215,7 @@ int sve_exception_el(CPUARMState *env, int el) /* * Given that SVE is enabled, return the vector length for EL. */ -uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) +uint32_t sve_vqm1_for_el(CPUARMState *env, int el) { ARMCPU *cpu = env_archcpu(env); uint32_t len = cpu->sve_max_vq - 1; @@ -6238,7 +6238,7 @@ static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { int cur_el = arm_current_el(env); - int old_len = sve_zcr_len_for_el(env, cur_el); + int old_len = sve_vqm1_for_el(env, cur_el); int new_len; /* Bits other than [3:0] are RAZ/WI. */ @@ -6249,7 +6249,7 @@ static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, * Because we arrived here, we know both FP and SVE are enabled; * otherwise we would have trapped access to the ZCR_ELn register. */ - new_len = sve_zcr_len_for_el(env, cur_el); + new_len = sve_vqm1_for_el(env, cur_el); if (new_len < old_len) { aarch64_sve_narrow_vq(env, new_len + 1); } @@ -11168,7 +11168,7 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, sve_el = 0; } } else if (sve_el == 0) { - DP_TBFLAG_A64(flags, VL, sve_zcr_len_for_el(env, el)); + DP_TBFLAG_A64(flags, VL, sve_vqm1_for_el(env, el)); } DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); } @@ -11534,10 +11534,10 @@ void aarch64_sve_change_el(CPUARMState *env, int old_el, */ old_a64 = old_el ? arm_el_is_aa64(env, old_el) : el0_a64; old_len = (old_a64 && !sve_exception_el(env, old_el) - ? sve_zcr_len_for_el(env, old_el) : 0); + ? sve_vqm1_for_el(env, old_el) : 0); new_a64 = new_el ? arm_el_is_aa64(env, new_el) : el0_a64; new_len = (new_a64 && !sve_exception_el(env, new_el) - ? sve_zcr_len_for_el(env, new_el) : 0); + ? sve_vqm1_for_el(env, new_el) : 0); /* When changing vector length, clear inaccessible state. */ if (new_len < old_len) { From d1ce862602ac23de7b3e8541507fc78a8c4a5766 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 48/55] target/arm: Split out load/store primitives to sve_ldst_internal.h Begin creation of sve_ldst_internal.h by moving the primitives that access host and tlb memory. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-14-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 107 +-------------------------- target/arm/sve_ldst_internal.h | 127 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 106 deletions(-) create mode 100644 target/arm/sve_ldst_internal.h diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index 3bdcd4ce9d..0c6dde00aa 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -21,12 +21,12 @@ #include "cpu.h" #include "internals.h" #include "exec/exec-all.h" -#include "exec/cpu_ldst.h" #include "exec/helper-proto.h" #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" #include "tcg/tcg.h" #include "vec_internal.h" +#include "sve_ldst_internal.h" /* Return a value for NZCV as per the ARM PredTest pseudofunction. @@ -5301,111 +5301,6 @@ void HELPER(sve_fcmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va, * Load contiguous data, protected by a governing predicate. */ -/* - * Load one element into @vd + @reg_off from @host. - * The controlling predicate is known to be true. - */ -typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host); - -/* - * Load one element into @vd + @reg_off from (@env, @vaddr, @ra). - * The controlling predicate is known to be true. - */ -typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off, - target_ulong vaddr, uintptr_t retaddr); - -/* - * Generate the above primitives. - */ - -#define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \ -static void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ -{ \ - TYPEM val = HOST(host); \ - *(TYPEE *)(vd + H(reg_off)) = val; \ -} - -#define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \ -static void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ -{ HOST(host, (TYPEM)*(TYPEE *)(vd + H(reg_off))); } - -#define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \ -static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off, \ - target_ulong addr, uintptr_t ra) \ -{ \ - *(TYPEE *)(vd + H(reg_off)) = \ - (TYPEM)TLB(env, useronly_clean_ptr(addr), ra); \ -} - -#define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \ -static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off, \ - target_ulong addr, uintptr_t ra) \ -{ \ - TLB(env, useronly_clean_ptr(addr), \ - (TYPEM)*(TYPEE *)(vd + H(reg_off)), ra); \ -} - -#define DO_LD_PRIM_1(NAME, H, TE, TM) \ - DO_LD_HOST(NAME, H, TE, TM, ldub_p) \ - DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra) - -DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t) -DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t) -DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t) -DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t) -DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t) -DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t) -DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t) - -#define DO_ST_PRIM_1(NAME, H, TE, TM) \ - DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \ - DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra) - -DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t) -DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t) -DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t) -DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t) - -#define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \ - DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \ - DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \ - DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \ - DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra) - -#define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \ - DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \ - DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \ - DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \ - DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra) - -DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw) -DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw) -DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw) -DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw) -DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw) - -DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw) -DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw) -DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw) - -DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl) -DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl) -DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl) - -DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl) -DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl) - -DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq) -DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) - -#undef DO_LD_TLB -#undef DO_ST_TLB -#undef DO_LD_HOST -#undef DO_LD_PRIM_1 -#undef DO_ST_PRIM_1 -#undef DO_LD_PRIM_2 -#undef DO_ST_PRIM_2 - /* * Skip through a sequence of inactive elements in the guarding predicate @vg, * beginning at @reg_off bounded by @reg_max. Return the offset of the active diff --git a/target/arm/sve_ldst_internal.h b/target/arm/sve_ldst_internal.h new file mode 100644 index 0000000000..ef9117e84c --- /dev/null +++ b/target/arm/sve_ldst_internal.h @@ -0,0 +1,127 @@ +/* + * ARM SVE Load/Store Helpers + * + * Copyright (c) 2018-2022 Linaro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef TARGET_ARM_SVE_LDST_INTERNAL_H +#define TARGET_ARM_SVE_LDST_INTERNAL_H + +#include "exec/cpu_ldst.h" + +/* + * Load one element into @vd + @reg_off from @host. + * The controlling predicate is known to be true. + */ +typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host); + +/* + * Load one element into @vd + @reg_off from (@env, @vaddr, @ra). + * The controlling predicate is known to be true. + */ +typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off, + target_ulong vaddr, uintptr_t retaddr); + +/* + * Generate the above primitives. + */ + +#define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \ +static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ +{ TYPEM val = HOST(host); *(TYPEE *)(vd + H(reg_off)) = val; } + +#define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \ +static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ +{ TYPEM val = *(TYPEE *)(vd + H(reg_off)); HOST(host, val); } + +#define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \ +static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ + intptr_t reg_off, target_ulong addr, uintptr_t ra) \ +{ \ + TYPEM val = TLB(env, useronly_clean_ptr(addr), ra); \ + *(TYPEE *)(vd + H(reg_off)) = val; \ +} + +#define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \ +static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ + intptr_t reg_off, target_ulong addr, uintptr_t ra) \ +{ \ + TYPEM val = *(TYPEE *)(vd + H(reg_off)); \ + TLB(env, useronly_clean_ptr(addr), val, ra); \ +} + +#define DO_LD_PRIM_1(NAME, H, TE, TM) \ + DO_LD_HOST(NAME, H, TE, TM, ldub_p) \ + DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra) + +DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t) +DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t) +DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t) +DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t) +DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t) +DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t) +DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t) + +#define DO_ST_PRIM_1(NAME, H, TE, TM) \ + DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \ + DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra) + +DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t) +DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t) +DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t) +DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t) + +#define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \ + DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \ + DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \ + DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \ + DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra) + +#define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \ + DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \ + DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \ + DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \ + DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra) + +DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw) +DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw) +DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw) +DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw) +DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw) + +DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw) +DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw) +DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw) + +DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl) +DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl) +DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl) + +DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl) +DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl) + +DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq) +DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) + +#undef DO_LD_TLB +#undef DO_ST_TLB +#undef DO_LD_HOST +#undef DO_LD_PRIM_1 +#undef DO_ST_PRIM_1 +#undef DO_LD_PRIM_2 +#undef DO_ST_PRIM_2 + +#endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */ From 0b68112b399d77c765a0a5d9bb4f4b4e57541ee7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 49/55] target/arm: Export sve contiguous ldst support functions Export all of the support functions for performing bulk fault analysis on a set of elements at contiguous addresses controlled by a predicate. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-15-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 87 ++++++------------------------- target/arm/sve_ldst_internal.h | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 70 deletions(-) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index 0c6dde00aa..8cd371e3e3 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -5341,16 +5341,9 @@ static intptr_t find_next_active(uint64_t *vg, intptr_t reg_off, * exit via page fault exception. */ -typedef struct { - void *host; - int flags; - MemTxAttrs attrs; -} SVEHostPage; - -static bool sve_probe_page(SVEHostPage *info, bool nofault, - CPUARMState *env, target_ulong addr, - int mem_off, MMUAccessType access_type, - int mmu_idx, uintptr_t retaddr) +bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, + target_ulong addr, int mem_off, MMUAccessType access_type, + int mmu_idx, uintptr_t retaddr) { int flags; @@ -5406,59 +5399,13 @@ static bool sve_probe_page(SVEHostPage *info, bool nofault, return true; } - -/* - * Analyse contiguous data, protected by a governing predicate. - */ - -typedef enum { - FAULT_NO, - FAULT_FIRST, - FAULT_ALL, -} SVEContFault; - -typedef struct { - /* - * First and last element wholly contained within the two pages. - * mem_off_first[0] and reg_off_first[0] are always set >= 0. - * reg_off_last[0] may be < 0 if the first element crosses pages. - * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1] - * are set >= 0 only if there are complete elements on a second page. - * - * The reg_off_* offsets are relative to the internal vector register. - * The mem_off_first offset is relative to the memory address; the - * two offsets are different when a load operation extends, a store - * operation truncates, or for multi-register operations. - */ - int16_t mem_off_first[2]; - int16_t reg_off_first[2]; - int16_t reg_off_last[2]; - - /* - * One element that is misaligned and spans both pages, - * or -1 if there is no such active element. - */ - int16_t mem_off_split; - int16_t reg_off_split; - - /* - * The byte offset at which the entire operation crosses a page boundary. - * Set >= 0 if and only if the entire operation spans two pages. - */ - int16_t page_split; - - /* TLB data for the two pages. */ - SVEHostPage page[2]; -} SVEContLdSt; - /* * Find first active element on each page, and a loose bound for the * final element on each page. Identify any single element that spans * the page boundary. Return true if there are any active elements. */ -static bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, - uint64_t *vg, intptr_t reg_max, - int esz, int msize) +bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, + intptr_t reg_max, int esz, int msize) { const int esize = 1 << esz; const uint64_t pg_mask = pred_esz_masks[esz]; @@ -5548,9 +5495,9 @@ static bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, * Control the generation of page faults with @fault. Return false if * there is no work to do, which can only happen with @fault == FAULT_NO. */ -static bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, - CPUARMState *env, target_ulong addr, - MMUAccessType access_type, uintptr_t retaddr) +bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, + CPUARMState *env, target_ulong addr, + MMUAccessType access_type, uintptr_t retaddr) { int mmu_idx = cpu_mmu_index(env, false); int mem_off = info->mem_off_first[0]; @@ -5606,12 +5553,12 @@ static bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, return have_work; } -static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, int wp_access, - uintptr_t retaddr) -{ #ifndef CONFIG_USER_ONLY +void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, + int esize, int msize, int wp_access, + uintptr_t retaddr) +{ intptr_t mem_off, reg_off, reg_last; int flags0 = info->page[0].flags; int flags1 = info->page[1].flags; @@ -5667,12 +5614,12 @@ static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, } while (reg_off & 63); } while (reg_off <= reg_last); } -#endif } +#endif -static void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, int esize, - int msize, uint32_t mtedesc, uintptr_t ra) +void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, int esize, + int msize, uint32_t mtedesc, uintptr_t ra) { intptr_t mem_off, reg_off, reg_last; diff --git a/target/arm/sve_ldst_internal.h b/target/arm/sve_ldst_internal.h index ef9117e84c..b5c473fc48 100644 --- a/target/arm/sve_ldst_internal.h +++ b/target/arm/sve_ldst_internal.h @@ -124,4 +124,98 @@ DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) #undef DO_LD_PRIM_2 #undef DO_ST_PRIM_2 +/* + * Resolve the guest virtual address to info->host and info->flags. + * If @nofault, return false if the page is invalid, otherwise + * exit via page fault exception. + */ + +typedef struct { + void *host; + int flags; + MemTxAttrs attrs; +} SVEHostPage; + +bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, + target_ulong addr, int mem_off, MMUAccessType access_type, + int mmu_idx, uintptr_t retaddr); + +/* + * Analyse contiguous data, protected by a governing predicate. + */ + +typedef enum { + FAULT_NO, + FAULT_FIRST, + FAULT_ALL, +} SVEContFault; + +typedef struct { + /* + * First and last element wholly contained within the two pages. + * mem_off_first[0] and reg_off_first[0] are always set >= 0. + * reg_off_last[0] may be < 0 if the first element crosses pages. + * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1] + * are set >= 0 only if there are complete elements on a second page. + * + * The reg_off_* offsets are relative to the internal vector register. + * The mem_off_first offset is relative to the memory address; the + * two offsets are different when a load operation extends, a store + * operation truncates, or for multi-register operations. + */ + int16_t mem_off_first[2]; + int16_t reg_off_first[2]; + int16_t reg_off_last[2]; + + /* + * One element that is misaligned and spans both pages, + * or -1 if there is no such active element. + */ + int16_t mem_off_split; + int16_t reg_off_split; + + /* + * The byte offset at which the entire operation crosses a page boundary. + * Set >= 0 if and only if the entire operation spans two pages. + */ + int16_t page_split; + + /* TLB data for the two pages. */ + SVEHostPage page[2]; +} SVEContLdSt; + +/* + * Find first active element on each page, and a loose bound for the + * final element on each page. Identify any single element that spans + * the page boundary. Return true if there are any active elements. + */ +bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, + intptr_t reg_max, int esz, int msize); + +/* + * Resolve the guest virtual addresses to info->page[]. + * Control the generation of page faults with @fault. Return false if + * there is no work to do, which can only happen with @fault == FAULT_NO. + */ +bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, + CPUARMState *env, target_ulong addr, + MMUAccessType access_type, uintptr_t retaddr); + +#ifdef CONFIG_USER_ONLY +static inline void +sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, + target_ulong addr, int esize, int msize, + int wp_access, uintptr_t retaddr) +{ } +#else +void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, + int esize, int msize, int wp_access, + uintptr_t retaddr); +#endif + +void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, + target_ulong addr, int esize, int msize, + uint32_t mtedesc, uintptr_t ra); + #endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */ From 820e0bb9ce1941af82d14ec4a981792cb3697956 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 50/55] target/arm: Move expand_pred_b to vec_internal.h Put the inline function near the array declaration. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-16-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 9 --------- target/arm/vec_internal.h | 8 +++++++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index 8cd371e3e3..e865c12527 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -103,15 +103,6 @@ uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words) return flags; } -/* - * Expand active predicate bits to bytes, for byte elements. - * (The data table itself is in vec_helper.c as MVE also needs it.) - */ -static inline uint64_t expand_pred_b(uint8_t byte) -{ - return expand_pred_b_data[byte]; -} - /* Similarly for half-word elements. * for (i = 0; i < 256; ++i) { * unsigned long m = 0; diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index 1d63402042..d1a1ea4a66 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -50,8 +50,14 @@ #define H8(x) (x) #define H1_8(x) (x) -/* Data for expanding active predicate bits to bytes, for byte elements. */ +/* + * Expand active predicate bits to bytes, for byte elements. + */ extern const uint64_t expand_pred_b_data[256]; +static inline uint64_t expand_pred_b(uint8_t byte) +{ + return expand_pred_b_data[byte]; +} static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) { From 05dd14bdfa89ffeb1a82b08776687d44b538185b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 51/55] target/arm: Use expand_pred_b in mve_helper.c Use the function instead of the array directly. Because the function performs its own masking, via the uint8_t parameter, we need to do nothing extra within the users: the bits above the first 2 (_uh) or 4 (_uw) will be discarded by assignment to the local bmask variables, and of course _uq uses the entire uint64_t result. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-17-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/mve_helper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/arm/mve_helper.c b/target/arm/mve_helper.c index 846962bf4c..403b345ea3 100644 --- a/target/arm/mve_helper.c +++ b/target/arm/mve_helper.c @@ -726,7 +726,7 @@ static void mergemask_sb(int8_t *d, int8_t r, uint16_t mask) static void mergemask_uh(uint16_t *d, uint16_t r, uint16_t mask) { - uint16_t bmask = expand_pred_b_data[mask & 3]; + uint16_t bmask = expand_pred_b(mask); *d = (*d & ~bmask) | (r & bmask); } @@ -737,7 +737,7 @@ static void mergemask_sh(int16_t *d, int16_t r, uint16_t mask) static void mergemask_uw(uint32_t *d, uint32_t r, uint16_t mask) { - uint32_t bmask = expand_pred_b_data[mask & 0xf]; + uint32_t bmask = expand_pred_b(mask); *d = (*d & ~bmask) | (r & bmask); } @@ -748,7 +748,7 @@ static void mergemask_sw(int32_t *d, int32_t r, uint16_t mask) static void mergemask_uq(uint64_t *d, uint64_t r, uint16_t mask) { - uint64_t bmask = expand_pred_b_data[mask & 0xff]; + uint64_t bmask = expand_pred_b(mask); *d = (*d & ~bmask) | (r & bmask); } From a613cf2d4ae61fbcdf084103ea10de193bfbe169 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 52/55] target/arm: Move expand_pred_h to vec_internal.h Move the data to vec_helper.c and the inline to vec_internal.h. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-18-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 29 ----------------------------- target/arm/vec_helper.c | 26 ++++++++++++++++++++++++++ target/arm/vec_internal.h | 7 +++++++ 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index e865c12527..1654c0bbf9 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -103,35 +103,6 @@ uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words) return flags; } -/* Similarly for half-word elements. - * for (i = 0; i < 256; ++i) { - * unsigned long m = 0; - * if (i & 0xaa) { - * continue; - * } - * for (j = 0; j < 8; j += 2) { - * if ((i >> j) & 1) { - * m |= 0xfffful << (j << 3); - * } - * } - * printf("[0x%x] = 0x%016lx,\n", i, m); - * } - */ -static inline uint64_t expand_pred_h(uint8_t byte) -{ - static const uint64_t word[] = { - [0x01] = 0x000000000000ffff, [0x04] = 0x00000000ffff0000, - [0x05] = 0x00000000ffffffff, [0x10] = 0x0000ffff00000000, - [0x11] = 0x0000ffff0000ffff, [0x14] = 0x0000ffffffff0000, - [0x15] = 0x0000ffffffffffff, [0x40] = 0xffff000000000000, - [0x41] = 0xffff00000000ffff, [0x44] = 0xffff0000ffff0000, - [0x45] = 0xffff0000ffffffff, [0x50] = 0xffffffff00000000, - [0x51] = 0xffffffff0000ffff, [0x54] = 0xffffffffffff0000, - [0x55] = 0xffffffffffffffff, - }; - return word[byte & 0x55]; -} - /* Similarly for single word elements. */ static inline uint64_t expand_pred_s(uint8_t byte) { diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index 17fb158362..26c373e522 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -127,6 +127,32 @@ const uint64_t expand_pred_b_data[256] = { 0xffffffffffffffff, }; +/* + * Similarly for half-word elements. + * for (i = 0; i < 256; ++i) { + * unsigned long m = 0; + * if (i & 0xaa) { + * continue; + * } + * for (j = 0; j < 8; j += 2) { + * if ((i >> j) & 1) { + * m |= 0xfffful << (j << 3); + * } + * } + * printf("[0x%x] = 0x%016lx,\n", i, m); + * } + */ +const uint64_t expand_pred_h_data[0x55 + 1] = { + [0x01] = 0x000000000000ffff, [0x04] = 0x00000000ffff0000, + [0x05] = 0x00000000ffffffff, [0x10] = 0x0000ffff00000000, + [0x11] = 0x0000ffff0000ffff, [0x14] = 0x0000ffffffff0000, + [0x15] = 0x0000ffffffffffff, [0x40] = 0xffff000000000000, + [0x41] = 0xffff00000000ffff, [0x44] = 0xffff0000ffff0000, + [0x45] = 0xffff0000ffffffff, [0x50] = 0xffffffff00000000, + [0x51] = 0xffffffff0000ffff, [0x54] = 0xffffffffffff0000, + [0x55] = 0xffffffffffffffff, +}; + /* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */ int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3, bool neg, bool round) diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index d1a1ea4a66..1d527fadac 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -59,6 +59,13 @@ static inline uint64_t expand_pred_b(uint8_t byte) return expand_pred_b_data[byte]; } +/* Similarly for half-word elements. */ +extern const uint64_t expand_pred_h_data[0x55 + 1]; +static inline uint64_t expand_pred_h(uint8_t byte) +{ + return expand_pred_h_data[byte & 0x55]; +} + static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) { uint64_t *d = vd + opr_sz; From 72db2aa353a98bd3486c9e5ef7e31ae7cf934849 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 53/55] target/arm: Export bfdotadd from vec_helper.c We will need this over in sme_helper.c. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-19-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/vec_helper.c | 2 +- target/arm/vec_internal.h | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index 26c373e522..9a9c034e36 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -2557,7 +2557,7 @@ DO_MMLA_B(gvec_usmmla_b, do_usmmla_b) * BFloat16 Dot Product */ -static float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2) +float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2) { /* FPCR is ignored for BFDOT and BFMMLA. */ float_status bf_status = { diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index 1d527fadac..1f4ed80ff7 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -230,4 +230,17 @@ uint64_t pmull_h(uint64_t op1, uint64_t op2); */ uint64_t pmull_w(uint64_t op1, uint64_t op2); +/** + * bfdotadd: + * @sum: addend + * @e1, @e2: multiplicand vectors + * + * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum. + * The @e1 and @e2 operands correspond to the 32-bit source vector + * slots and contain two Bfloat16 values each. + * + * Corresponds to the ARM pseudocode function BFDotAdd. + */ +float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2); + #endif /* TARGET_ARM_VEC_INTERNAL_H */ From f305bf9436896b4cd9ef622034e166b024780874 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:59 +0100 Subject: [PATCH 54/55] target/arm: Add isar_feature_aa64_sme This will be used for implementing FEAT_SME. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-20-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index e45b5cb7fe..2e6153c540 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -4048,6 +4048,11 @@ static inline bool isar_feature_aa64_mte(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, MTE) >= 2; } +static inline bool isar_feature_aa64_sme(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SME) != 0; +} + static inline bool isar_feature_aa64_pmu_8_1(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) >= 4 && From 414c54d515dba16bfaef643a8acec200c05f229a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:59 +0100 Subject: [PATCH 55/55] target/arm: Add ID_AA64SMFR0_EL1 This register is allocated from the existing block of id registers, so it is already RES0 for cpus that do not implement SME. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-21-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 25 +++++++++++++++++++++++++ target/arm/helper.c | 4 ++-- target/arm/kvm64.c | 11 +++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 2e6153c540..78dbcb5592 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -966,6 +966,7 @@ struct ArchCPU { uint64_t id_aa64dfr0; uint64_t id_aa64dfr1; uint64_t id_aa64zfr0; + uint64_t id_aa64smfr0; uint64_t reset_pmcr_el0; } isar; uint64_t midr; @@ -2190,6 +2191,15 @@ FIELD(ID_AA64ZFR0, I8MM, 44, 4) FIELD(ID_AA64ZFR0, F32MM, 52, 4) FIELD(ID_AA64ZFR0, F64MM, 56, 4) +FIELD(ID_AA64SMFR0, F32F32, 32, 1) +FIELD(ID_AA64SMFR0, B16F32, 34, 1) +FIELD(ID_AA64SMFR0, F16F32, 35, 1) +FIELD(ID_AA64SMFR0, I8I32, 36, 4) +FIELD(ID_AA64SMFR0, F64F64, 48, 1) +FIELD(ID_AA64SMFR0, I16I64, 52, 4) +FIELD(ID_AA64SMFR0, SMEVER, 56, 4) +FIELD(ID_AA64SMFR0, FA64, 63, 1) + FIELD(ID_DFR0, COPDBG, 0, 4) FIELD(ID_DFR0, COPSDBG, 4, 4) FIELD(ID_DFR0, MMAPDBG, 8, 4) @@ -4195,6 +4205,21 @@ static inline bool isar_feature_aa64_sve_f64mm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F64MM) != 0; } +static inline bool isar_feature_aa64_sme_f64f64(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, F64F64); +} + +static inline bool isar_feature_aa64_sme_i16i64(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, I16I64) == 0xf; +} + +static inline bool isar_feature_aa64_sme_fa64(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, FA64); +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper.c b/target/arm/helper.c index 400f7cd1db..ac9942d750 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -7722,11 +7722,11 @@ void register_cp_regs_for_features(ARMCPU *cpu) .access = PL1_R, .type = ARM_CP_CONST, .accessfn = access_aa64_tid3, .resetvalue = cpu->isar.id_aa64zfr0 }, - { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64, + { .name = "ID_AA64SMFR0_EL1", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5, .access = PL1_R, .type = ARM_CP_CONST, .accessfn = access_aa64_tid3, - .resetvalue = 0 }, + .resetvalue = cpu->isar.id_aa64smfr0 }, { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST, diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index b3f635fc95..ff8f65da22 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -574,6 +574,8 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) } else { err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1, ARM64_SYS_REG(3, 0, 0, 4, 1)); + err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64smfr0, + ARM64_SYS_REG(3, 0, 0, 4, 5)); err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0, ARM64_SYS_REG(3, 0, 0, 5, 0)); err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1, @@ -682,10 +684,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) ahcf->isar.id_aa64pfr0 = t; /* - * Before v5.1, KVM did not support SVE and did not expose - * ID_AA64ZFR0_EL1 even as RAZ. After v5.1, KVM still does - * not expose the register to "user" requests like this - * unless the host supports SVE. + * There is a range of kernels between kernel commit 73433762fcae + * and f81cb2c3ad41 which have a bug where the kernel doesn't expose + * SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has enabled + * SVE support, so we only read it here, rather than together with all + * the other ID registers earlier. */ err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0, ARM64_SYS_REG(3, 0, 0, 4, 4));