From 298c31de9871b971b86356164633b9a5861af3db Mon Sep 17 00:00:00 2001 From: John Millikin Date: Thu, 4 Aug 2022 21:29:51 +0900 Subject: [PATCH 01/37] scsi-disk: support setting CD-ROM block size via device options SunOS expects CD-ROM devices to have a block size of 512, and will fail to mount or install using QEMU's default block size of 2048. When initializing the SCSI device, allow the `physical_block_size' block device option to override the default block size. Signed-off-by: John Millikin Message-Id: <20220804122950.1577012-1-john@john-millikin.com> Signed-off-by: Paolo Bonzini --- hw/scsi/scsi-disk.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c index 399e1787ea..e493c28814 100644 --- a/hw/scsi/scsi-disk.c +++ b/hw/scsi/scsi-disk.c @@ -2544,6 +2544,7 @@ static void scsi_cd_realize(SCSIDevice *dev, Error **errp) SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev); AioContext *ctx; int ret; + uint32_t blocksize = 2048; if (!dev->conf.blk) { /* Anonymous BlockBackend for an empty drive. As we put it into @@ -2553,9 +2554,13 @@ static void scsi_cd_realize(SCSIDevice *dev, Error **errp) assert(ret == 0); } + if (dev->conf.physical_block_size != 0) { + blocksize = dev->conf.physical_block_size; + } + ctx = blk_get_aio_context(dev->conf.blk); aio_context_acquire(ctx); - s->qdev.blocksize = 2048; + s->qdev.blocksize = blocksize; s->qdev.type = TYPE_ROM; s->features |= 1 << SCSI_DISK_F_REMOVABLE; if (!s->product) { From 12f89a39cf3c5760cba82ce68929d748961f62df Mon Sep 17 00:00:00 2001 From: Chenyi Qiang Date: Thu, 29 Sep 2022 15:20:11 +0800 Subject: [PATCH 02/37] i386: kvm: extend kvm_{get, put}_vcpu_events to support pending triple fault For the direct triple faults, i.e. hardware detected and KVM morphed to VM-Exit, KVM will never lose them. But for triple faults sythesized by KVM, e.g. the RSM path, if KVM exits to userspace before the request is serviced, userspace could migrate the VM and lose the triple fault. A new flag KVM_VCPUEVENT_VALID_TRIPLE_FAULT is defined to signal that the event.triple_fault_pending field contains a valid state if the KVM_CAP_X86_TRIPLE_FAULT_EVENT capability is enabled. Acked-by: Peter Xu Signed-off-by: Chenyi Qiang Message-Id: <20220929072014.20705-2-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini --- target/i386/cpu.c | 1 + target/i386/cpu.h | 1 + target/i386/kvm/kvm.c | 20 ++++++++++++++++++++ target/i386/machine.c | 20 ++++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index ad623d91e4..06884177fa 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -6017,6 +6017,7 @@ static void x86_cpu_reset(DeviceState *dev) env->exception_has_payload = false; env->exception_payload = 0; env->nmi_injected = false; + env->triple_fault_pending = false; #if !defined(CONFIG_USER_ONLY) /* We hard-wire the BSP to the first CPU. */ apic_designate_bsp(cpu->apic_state, s->cpu_index == 0); diff --git a/target/i386/cpu.h b/target/i386/cpu.h index 82004b65b9..d4124973ce 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -1739,6 +1739,7 @@ typedef struct CPUArchState { uint8_t has_error_code; uint8_t exception_has_payload; uint64_t exception_payload; + uint8_t triple_fault_pending; uint32_t ins_len; uint32_t sipi_vector; bool tsc_valid; diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index f2a96492ce..3ebe8b7f1f 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -132,6 +132,7 @@ static int has_xcrs; static int has_pit_state2; static int has_sregs2; static int has_exception_payload; +static int has_triple_fault_event; static bool has_msr_mcg_ext_ctl; @@ -2479,6 +2480,16 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + has_triple_fault_event = kvm_check_extension(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT); + if (has_triple_fault_event) { + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT, 0, true); + if (ret < 0) { + error_report("kvm: Failed to enable triple fault event cap: %s", + strerror(-ret)); + return ret; + } + } + ret = kvm_get_supported_msrs(s); if (ret < 0) { return ret; @@ -4295,6 +4306,11 @@ static int kvm_put_vcpu_events(X86CPU *cpu, int level) } } + if (has_triple_fault_event) { + events.flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT; + events.triple_fault.pending = env->triple_fault_pending; + } + return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events); } @@ -4364,6 +4380,10 @@ static int kvm_get_vcpu_events(X86CPU *cpu) } } + if (events.flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) { + env->triple_fault_pending = events.triple_fault.pending; + } + env->sipi_vector = events.sipi_vector; return 0; diff --git a/target/i386/machine.c b/target/i386/machine.c index cecd476e98..310b125235 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -1562,6 +1562,25 @@ static const VMStateDescription vmstate_arch_lbr = { } }; +static bool triple_fault_needed(void *opaque) +{ + X86CPU *cpu = opaque; + CPUX86State *env = &cpu->env; + + return env->triple_fault_pending; +} + +static const VMStateDescription vmstate_triple_fault = { + .name = "cpu/triple_fault", + .version_id = 1, + .minimum_version_id = 1, + .needed = triple_fault_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT8(env.triple_fault_pending, X86CPU), + VMSTATE_END_OF_LIST() + } +}; + const VMStateDescription vmstate_x86_cpu = { .name = "cpu", .version_id = 12, @@ -1706,6 +1725,7 @@ const VMStateDescription vmstate_x86_cpu = { &vmstate_amx_xtile, #endif &vmstate_arch_lbr, + &vmstate_triple_fault, NULL } }; From 3dba0a335cf5c53146b606be6ddfab4df81c464e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 29 Sep 2022 15:20:12 +0800 Subject: [PATCH 03/37] kvm: allow target-specific accelerator properties Several hypervisor capabilities in KVM are target-specific. When exposed to QEMU users as accelerator properties (i.e. -accel kvm,prop=value), they should not be available for all targets. Add a hook for targets to add their own properties to -accel kvm, for now no such property is defined. Signed-off-by: Paolo Bonzini Message-Id: <20220929072014.20705-3-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini --- accel/kvm/kvm-all.c | 2 ++ include/sysemu/kvm.h | 2 ++ target/arm/kvm.c | 4 ++++ target/i386/kvm/kvm.c | 4 ++++ target/mips/kvm.c | 4 ++++ target/ppc/kvm.c | 4 ++++ target/riscv/kvm.c | 4 ++++ target/s390x/kvm/kvm.c | 4 ++++ 8 files changed, 28 insertions(+) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 423fb1936f..03a69cf053 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -3731,6 +3731,8 @@ static void kvm_accel_class_init(ObjectClass *oc, void *data) NULL, NULL); object_class_property_set_description(oc, "dirty-ring-size", "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); + + kvm_arch_accel_class_init(oc); } static const TypeInfo kvm_accel_type = { diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index 790d35ef78..e9a97eda8c 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -349,6 +349,8 @@ bool kvm_device_supported(int vmfd, uint64_t type); extern const KVMCapabilityInfo kvm_arch_required_capabilities[]; +void kvm_arch_accel_class_init(ObjectClass *oc); + void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run); MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run); diff --git a/target/arm/kvm.c b/target/arm/kvm.c index e5c1bd50d2..d21603cf28 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -1056,3 +1056,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 3ebe8b7f1f..f18d21413c 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -5468,3 +5468,7 @@ void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask) mask &= ~BIT_ULL(bit); } } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/mips/kvm.c b/target/mips/kvm.c index caf70decd2..bcb8e06b2c 100644 --- a/target/mips/kvm.c +++ b/target/mips/kvm.c @@ -1294,3 +1294,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c index 466d0d2f4c..7c25348b7b 100644 --- a/target/ppc/kvm.c +++ b/target/ppc/kvm.c @@ -2966,3 +2966,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c index 70b4cff06f..30f21453d6 100644 --- a/target/riscv/kvm.c +++ b/target/riscv/kvm.c @@ -532,3 +532,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c index 6a8dbadf7e..508c24cfec 100644 --- a/target/s390x/kvm/kvm.c +++ b/target/s390x/kvm/kvm.c @@ -2581,3 +2581,7 @@ int kvm_s390_get_zpci_op(void) { return cap_zpci_op; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} From 5f8a6bce1f1080058ed29d716cae81ea805142ae Mon Sep 17 00:00:00 2001 From: Chenyi Qiang Date: Thu, 29 Sep 2022 15:20:13 +0800 Subject: [PATCH 04/37] kvm: expose struct KVMState Expose struct KVMState out of kvm-all.c so that the field of struct KVMState can be accessed when defining target-specific accelerator properties. Signed-off-by: Chenyi Qiang Message-Id: <20220929072014.20705-4-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini --- accel/kvm/kvm-all.c | 74 -------------------------------------- include/sysemu/kvm_int.h | 76 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 74 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 03a69cf053..fbfe948398 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -77,86 +77,12 @@ do { } while (0) #endif -#define KVM_MSI_HASHTAB_SIZE 256 - struct KVMParkedVcpu { unsigned long vcpu_id; int kvm_fd; QLIST_ENTRY(KVMParkedVcpu) node; }; -enum KVMDirtyRingReaperState { - KVM_DIRTY_RING_REAPER_NONE = 0, - /* The reaper is sleeping */ - KVM_DIRTY_RING_REAPER_WAIT, - /* The reaper is reaping for dirty pages */ - KVM_DIRTY_RING_REAPER_REAPING, -}; - -/* - * KVM reaper instance, responsible for collecting the KVM dirty bits - * via the dirty ring. - */ -struct KVMDirtyRingReaper { - /* The reaper thread */ - QemuThread reaper_thr; - volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ - volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ -}; - -struct KVMState -{ - AccelState parent_obj; - - int nr_slots; - int fd; - int vmfd; - int coalesced_mmio; - int coalesced_pio; - struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; - bool coalesced_flush_in_progress; - int vcpu_events; - int robust_singlestep; - int debugregs; -#ifdef KVM_CAP_SET_GUEST_DEBUG - QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; -#endif - int max_nested_state_len; - int many_ioeventfds; - int intx_set_mask; - int kvm_shadow_mem; - bool kernel_irqchip_allowed; - bool kernel_irqchip_required; - OnOffAuto kernel_irqchip_split; - bool sync_mmu; - uint64_t manual_dirty_log_protect; - /* The man page (and posix) say ioctl numbers are signed int, but - * they're not. Linux, glibc and *BSD all treat ioctl numbers as - * unsigned, and treating them as signed here can break things */ - unsigned irq_set_ioctl; - unsigned int sigmask_len; - GHashTable *gsimap; -#ifdef KVM_CAP_IRQ_ROUTING - struct kvm_irq_routing *irq_routes; - int nr_allocated_irq_routes; - unsigned long *used_gsi_bitmap; - unsigned int gsi_count; - QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; -#endif - KVMMemoryListener memory_listener; - QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; - - /* For "info mtree -f" to tell if an MR is registered in KVM */ - int nr_as; - struct KVMAs { - KVMMemoryListener *ml; - AddressSpace *as; - } *as; - uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ - uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ - struct KVMDirtyRingReaper reaper; -}; - KVMState *kvm_state; bool kvm_kernel_irqchip; bool kvm_split_irqchip; diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index 1f5487d9b7..3b4adcdc10 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -10,6 +10,7 @@ #define QEMU_KVM_INT_H #include "exec/memory.h" +#include "qapi/qapi-types-common.h" #include "qemu/accel.h" #include "sysemu/kvm.h" @@ -36,6 +37,81 @@ typedef struct KVMMemoryListener { int as_id; } KVMMemoryListener; +#define KVM_MSI_HASHTAB_SIZE 256 + +enum KVMDirtyRingReaperState { + KVM_DIRTY_RING_REAPER_NONE = 0, + /* The reaper is sleeping */ + KVM_DIRTY_RING_REAPER_WAIT, + /* The reaper is reaping for dirty pages */ + KVM_DIRTY_RING_REAPER_REAPING, +}; + +/* + * KVM reaper instance, responsible for collecting the KVM dirty bits + * via the dirty ring. + */ +struct KVMDirtyRingReaper { + /* The reaper thread */ + QemuThread reaper_thr; + volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ + volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ +}; +struct KVMState +{ + AccelState parent_obj; + + int nr_slots; + int fd; + int vmfd; + int coalesced_mmio; + int coalesced_pio; + struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; + bool coalesced_flush_in_progress; + int vcpu_events; + int robust_singlestep; + int debugregs; +#ifdef KVM_CAP_SET_GUEST_DEBUG + QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; +#endif + int max_nested_state_len; + int many_ioeventfds; + int intx_set_mask; + int kvm_shadow_mem; + bool kernel_irqchip_allowed; + bool kernel_irqchip_required; + OnOffAuto kernel_irqchip_split; + bool sync_mmu; + uint64_t manual_dirty_log_protect; + /* The man page (and posix) say ioctl numbers are signed int, but + * they're not. Linux, glibc and *BSD all treat ioctl numbers as + * unsigned, and treating them as signed here can break things */ + unsigned irq_set_ioctl; + unsigned int sigmask_len; + GHashTable *gsimap; +#ifdef KVM_CAP_IRQ_ROUTING + struct kvm_irq_routing *irq_routes; + int nr_allocated_irq_routes; + unsigned long *used_gsi_bitmap; + unsigned int gsi_count; + QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; +#endif + KVMMemoryListener memory_listener; + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; + + /* For "info mtree -f" to tell if an MR is registered in KVM */ + int nr_as; + struct KVMAs { + KVMMemoryListener *ml; + AddressSpace *as; + } *as; + uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ + uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ + struct KVMDirtyRingReaper reaper; + NotifyVmexitOption notify_vmexit; + uint32_t notify_window; +}; + void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, AddressSpace *as, int as_id, const char *name); From e2e69f6bb907a70ac518230c54e98e7abcb0c911 Mon Sep 17 00:00:00 2001 From: Chenyi Qiang Date: Thu, 29 Sep 2022 15:20:14 +0800 Subject: [PATCH 05/37] i386: add notify VM exit support There are cases that malicious virtual machine can cause CPU stuck (due to event windows don't open up), e.g., infinite loop in microcode when nested #AC (CVE-2015-5307). No event window means no event (NMI, SMI and IRQ) can be delivered. It leads the CPU to be unavailable to host or other VMs. Notify VM exit is introduced to mitigate such kind of attacks, which will generate a VM exit if no event window occurs in VM non-root mode for a specified amount of time (notify window). A new KVM capability KVM_CAP_X86_NOTIFY_VMEXIT is exposed to user space so that the user can query the capability and set the expected notify window when creating VMs. The format of the argument when enabling this capability is as follows: Bit 63:32 - notify window specified in qemu command Bit 31:0 - some flags (e.g. KVM_X86_NOTIFY_VMEXIT_ENABLED is set to enable the feature.) Users can configure the feature by a new (x86 only) accel property: qemu -accel kvm,notify-vmexit=run|internal-error|disable,notify-window=n The default option of notify-vmexit is run, which will enable the capability and do nothing if the exit happens. The internal-error option raises a KVM internal error if it happens. The disable option does not enable the capability. The default value of notify-window is 0. It is valid only when notify-vmexit is not disabled. The valid range of notify-window is non-negative. It is even safe to set it to zero since there's an internal hardware threshold to be added to ensure no false positive. Because a notify VM exit may happen with VM_CONTEXT_INVALID set in exit qualification (no cases are anticipated that would set this bit), which means VM context is corrupted. It would be reflected in the flags of KVM_EXIT_NOTIFY exit. If KVM_NOTIFY_CONTEXT_INVALID bit is set, raise a KVM internal error unconditionally. Acked-by: Peter Xu Signed-off-by: Chenyi Qiang Message-Id: <20220929072014.20705-5-chenyi.qiang@intel.com> Signed-off-by: Paolo Bonzini --- accel/kvm/kvm-all.c | 2 + qapi/run-state.json | 17 ++++++++ qemu-options.hx | 11 +++++ target/i386/kvm/kvm.c | 98 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index fbfe948398..f99b0becd8 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -3618,6 +3618,8 @@ static void kvm_accel_instance_init(Object *obj) s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; /* KVM dirty ring is by default off */ s->kvm_dirty_ring_size = 0; + s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; + s->notify_window = 0; } /** diff --git a/qapi/run-state.json b/qapi/run-state.json index 9273ea6516..49989d30e6 100644 --- a/qapi/run-state.json +++ b/qapi/run-state.json @@ -643,3 +643,20 @@ { 'struct': 'MemoryFailureFlags', 'data': { 'action-required': 'bool', 'recursive': 'bool'} } + +## +# @NotifyVmexitOption: +# +# An enumeration of the options specified when enabling notify VM exit +# +# @run: enable the feature, do nothing and continue if the notify VM exit happens. +# +# @internal-error: enable the feature, raise a internal error if the notify +# VM exit happens. +# +# @disable: disable the feature. +# +# Since: 7.2 +## +{ 'enum': 'NotifyVmexitOption', + 'data': [ 'run', 'internal-error', 'disable' ] } \ No newline at end of file diff --git a/qemu-options.hx b/qemu-options.hx index 95b998a13b..606d379186 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -191,6 +191,7 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel, " split-wx=on|off (enable TCG split w^x mapping)\n" " tb-size=n (TCG translation block cache size)\n" " dirty-ring-size=n (KVM dirty ring GFN count, default 0)\n" + " notify-vmexit=run|internal-error|disable,notify-window=n (enable notify VM exit and set notify window, x86 only)\n" " thread=single|multi (enable multi-threaded TCG)\n", QEMU_ARCH_ALL) SRST ``-accel name[,prop=value[,...]]`` @@ -242,6 +243,16 @@ SRST is disabled (dirty-ring-size=0). When enabled, KVM will instead record dirty pages in a bitmap. + ``notify-vmexit=run|internal-error|disable,notify-window=n`` + Enables or disables notify VM exit support on x86 host and specify + the corresponding notify window to trigger the VM exit if enabled. + ``run`` option enables the feature. It does nothing and continue + if the exit happens. ``internal-error`` option enables the feature. + It raises a internal error. ``disable`` option doesn't enable the feature. + This feature can mitigate the CPU stuck issue due to event windows don't + open up for a specified of time (i.e. notify-window). + Default: notify-vmexit=run,notify-window=0. + ERST DEF("smp", HAS_ARG, QEMU_OPTION_smp, diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index f18d21413c..ec63b5eb10 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -15,6 +15,7 @@ #include "qemu/osdep.h" #include "qapi/qapi-events-run-state.h" #include "qapi/error.h" +#include "qapi/visitor.h" #include #include #include @@ -2595,6 +2596,21 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + if (s->notify_vmexit != NOTIFY_VMEXIT_OPTION_DISABLE && + kvm_check_extension(s, KVM_CAP_X86_NOTIFY_VMEXIT)) { + uint64_t notify_window_flags = + ((uint64_t)s->notify_window << 32) | + KVM_X86_NOTIFY_VMEXIT_ENABLED | + KVM_X86_NOTIFY_VMEXIT_USER; + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_NOTIFY_VMEXIT, 0, + notify_window_flags); + if (ret < 0) { + error_report("kvm: Failed to enable notify vmexit cap: %s", + strerror(-ret)); + return ret; + } + } + return 0; } @@ -5137,6 +5153,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) X86CPU *cpu = X86_CPU(cs); uint64_t code; int ret; + bool ctx_invalid; + char str[256]; + KVMState *state; switch (run->exit_reason) { case KVM_EXIT_HLT: @@ -5192,6 +5211,21 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) /* already handled in kvm_arch_post_run */ ret = 0; break; + case KVM_EXIT_NOTIFY: + ctx_invalid = !!(run->notify.flags & KVM_NOTIFY_CONTEXT_INVALID); + state = KVM_STATE(current_accel()); + sprintf(str, "Encounter a notify exit with %svalid context in" + " guest. There can be possible misbehaves in guest." + " Please have a look.", ctx_invalid ? "in" : ""); + if (ctx_invalid || + state->notify_vmexit == NOTIFY_VMEXIT_OPTION_INTERNAL_ERROR) { + warn_report("KVM internal error: %s", str); + ret = -1; + } else { + warn_report_once("KVM: %s", str); + ret = 0; + } + break; default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); ret = -1; @@ -5469,6 +5503,70 @@ void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask) } } +static int kvm_arch_get_notify_vmexit(Object *obj, Error **errp) +{ + KVMState *s = KVM_STATE(obj); + return s->notify_vmexit; +} + +static void kvm_arch_set_notify_vmexit(Object *obj, int value, Error **errp) +{ + KVMState *s = KVM_STATE(obj); + + if (s->fd != -1) { + error_setg(errp, "Cannot set properties after the accelerator has been initialized"); + return; + } + + s->notify_vmexit = value; +} + +static void kvm_arch_get_notify_window(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + uint32_t value = s->notify_window; + + visit_type_uint32(v, name, &value, errp); +} + +static void kvm_arch_set_notify_window(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + Error *error = NULL; + uint32_t value; + + if (s->fd != -1) { + error_setg(errp, "Cannot set properties after the accelerator has been initialized"); + return; + } + + visit_type_uint32(v, name, &value, &error); + if (error) { + error_propagate(errp, error); + return; + } + + s->notify_window = value; +} + void kvm_arch_accel_class_init(ObjectClass *oc) { + object_class_property_add_enum(oc, "notify-vmexit", "NotifyVMexitOption", + &NotifyVmexitOption_lookup, + kvm_arch_get_notify_vmexit, + kvm_arch_set_notify_vmexit); + object_class_property_set_description(oc, "notify-vmexit", + "Enable notify VM exit"); + + object_class_property_add(oc, "notify-window", "uint32", + kvm_arch_get_notify_window, + kvm_arch_set_notify_window, + NULL, NULL); + object_class_property_set_description(oc, "notify-window", + "Clock cycles without an event window " + "after which a notification VM exit occurs"); } From ddf83b35bde680faecae693c0fa03c18814a7a2e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:10 -0700 Subject: [PATCH 06/37] target/i386: Remove pc_start The DisasContext member and the disas_insn local variable of the same name are identical to DisasContextBase.pc_next. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-2-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 114 +++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 44af8c107f..16bf56dbc7 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -76,7 +76,6 @@ typedef struct DisasContext { DisasContextBase base; target_ulong pc; /* pc = eip + cs_base */ - target_ulong pc_start; /* pc at TB entry */ target_ulong cs_base; /* base of CS segment */ MemOp aflag; @@ -1345,13 +1344,13 @@ static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip) the instruction is known, but it isn't allowed in the current cpu mode. */ static void gen_illegal_opcode(DisasContext *s) { - gen_exception(s, EXCP06_ILLOP, s->pc_start - s->cs_base); + gen_exception(s, EXCP06_ILLOP, s->base.pc_next - s->cs_base); } /* Generate #GP for the current instruction. */ static void gen_exception_gpf(DisasContext *s) { - gen_exception(s, EXCP0D_GPF, s->pc_start - s->cs_base); + gen_exception(s, EXCP0D_GPF, s->base.pc_next - s->cs_base); } /* Check for cpl == 0; if not, raise #GP and return false. */ @@ -2016,7 +2015,7 @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes) } s->pc += num_bytes; - if (unlikely(s->pc - s->pc_start > X86_MAX_INSN_LENGTH)) { + if (unlikely(s->pc - s->base.pc_next > X86_MAX_INSN_LENGTH)) { /* If the instruction's 16th byte is on a different page than the 1st, a * page fault on the second page wins over the general protection fault * caused by the instruction being too long. @@ -2614,7 +2613,7 @@ static void gen_unknown_opcode(CPUX86State *env, DisasContext *s) if (qemu_loglevel_mask(LOG_UNIMP)) { FILE *logfile = qemu_log_trylock(); if (logfile) { - target_ulong pc = s->pc_start, end = s->pc; + target_ulong pc = s->base.pc_next, end = s->pc; fprintf(logfile, "ILLOPC: " TARGET_FMT_lx ":", pc); for (; pc < end; ++pc) { @@ -3226,8 +3225,7 @@ static const struct SSEOpHelper_table7 sse_op_table7[256] = { goto illegal_op; \ } while (0) -static void gen_sse(CPUX86State *env, DisasContext *s, int b, - target_ulong pc_start) +static void gen_sse(CPUX86State *env, DisasContext *s, int b) { int b1, op1_offset, op2_offset, is_xmm, val; int modrm, mod, rm, reg; @@ -3269,7 +3267,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b, } /* simple MMX/SSE operation */ if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); return; } if (s->flags & HF_EM_MASK) { @@ -4717,11 +4715,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) MemOp ot, aflag, dflag; int modrm, reg, rm, mod, op, opreg, val; target_ulong next_eip, tval; - target_ulong pc_start = s->base.pc_next; bool orig_cc_op_dirty = s->cc_op_dirty; CCOp orig_cc_op = s->cc_op; - s->pc_start = s->pc = pc_start; + s->pc = s->base.pc_next; s->override = -1; #ifdef TARGET_X86_64 s->rex_w = false; @@ -4745,7 +4742,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) s->base.num_insns--; tcg_remove_ops_after(s->prev_insn_end); s->base.is_jmp = DISAS_TOO_MANY; - return pc_start; + return s->base.pc_next; default: g_assert_not_reached(); } @@ -6079,7 +6076,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (s->flags & (HF_EM_MASK | HF_TS_MASK)) { /* if CR0.EM or CR0.TS are set, generate an FPU exception */ /* XXX: what to do if illegal op ? */ - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); break; } modrm = x86_ldub_code(env, s); @@ -6620,7 +6617,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) offsetof(CPUX86State, segs[R_CS].selector)); tcg_gen_st16_i32(s->tmp2_i32, cpu_env, offsetof(CPUX86State, fpcs)); - tcg_gen_st_tl(tcg_constant_tl(pc_start - s->cs_base), + tcg_gen_st_tl(tcg_constant_tl(s->base.pc_next - s->cs_base), cpu_env, offsetof(CPUX86State, fpip)); } } @@ -6632,7 +6629,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xa5: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_movs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_movs(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base); } else { gen_movs(s, ot); } @@ -6642,7 +6640,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xab: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_stos(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_stos(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base); } else { gen_stos(s, ot); } @@ -6651,7 +6650,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xad: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_lods(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_lods(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base); } else { gen_lods(s, ot); } @@ -6660,9 +6660,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xaf: ot = mo_b_d(b, dflag); if (prefixes & PREFIX_REPNZ) { - gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1); + gen_repz_scas(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base, 1); } else if (prefixes & PREFIX_REPZ) { - gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0); + gen_repz_scas(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base, 0); } else { gen_scas(s, ot); } @@ -6672,9 +6674,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xa7: ot = mo_b_d(b, dflag); if (prefixes & PREFIX_REPNZ) { - gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1); + gen_repz_cmps(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base, 1); } else if (prefixes & PREFIX_REPZ) { - gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0); + gen_repz_cmps(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base, 0); } else { gen_cmps(s, ot); } @@ -6692,7 +6696,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_io_start(); } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_ins(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base); /* jump generated by gen_repz_ins */ } else { gen_ins(s, ot); @@ -6713,7 +6718,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_io_start(); } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_outs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_outs(s, ot, s->base.pc_next - s->cs_base, + s->pc - s->cs_base); /* jump generated by gen_repz_outs */ } else { gen_outs(s, ot); @@ -6825,7 +6831,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) do_lret: if (PE(s) && !VM86(s)) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_lret_protected(cpu_env, tcg_const_i32(dflag - 1), tcg_const_i32(val)); } else { @@ -7295,7 +7301,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; val = x86_ldub_code(env, s); if (val == 0) { - gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base); + gen_exception(s, EXCP00_DIVZ, s->base.pc_next - s->cs_base); } else { gen_helper_aam(cpu_env, tcg_const_i32(val)); set_cc_op(s, CC_OP_LOGICB); @@ -7321,34 +7327,34 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (prefixes & PREFIX_REPZ) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_pause(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_helper_pause(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; } break; case 0x9b: /* fwait */ if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); } else { gen_helper_fwait(cpu_env); } break; case 0xcc: /* int3 */ - gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base); + gen_interrupt(s, EXCP03_INT3, s->base.pc_next - s->cs_base, s->pc - s->cs_base); break; case 0xcd: /* int N */ val = x86_ldub_code(env, s); if (check_vm86_iopl(s)) { - gen_interrupt(s, val, pc_start - s->cs_base, s->pc - s->cs_base); + gen_interrupt(s, val, s->base.pc_next - s->cs_base, s->pc - s->cs_base); } break; case 0xce: /* into */ if (CODE64(s)) goto illegal_op; gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_into(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_helper_into(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); break; #ifdef WANT_ICEBP case 0xf1: /* icebp (undocumented, exits to external debugger) */ @@ -7454,7 +7460,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0x132: /* rdmsr */ if (check_cpl0(s)) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); if (b & 2) { gen_helper_rdmsr(cpu_env); } else { @@ -7466,7 +7472,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; case 0x131: /* rdtsc */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); } @@ -7477,7 +7483,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; case 0x133: /* rdpmc */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_rdpmc(cpu_env); s->base.is_jmp = DISAS_NORETURN; break; @@ -7507,8 +7513,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0x105: /* syscall */ /* XXX: is it usable in real mode ? */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_syscall(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_helper_syscall(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); /* TF handling for the syscall insn is different. The TF bit is checked after the syscall insn completes. This allows #DB to not be generated after one has entered CPL0 if TF is set in FMASK. */ @@ -7533,14 +7539,14 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) #endif case 0x1a2: /* cpuid */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_cpuid(cpu_env); break; case 0xf4: /* hlt */ if (check_cpl0(s)) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_hlt(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_helper_hlt(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; } break; @@ -7636,7 +7642,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); tcg_gen_mov_tl(s->A0, cpu_regs[R_EAX]); gen_extu(s->aflag, s->A0); gen_add_A0_ds_seg(s); @@ -7648,8 +7654,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; break; @@ -7726,9 +7732,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1), - tcg_const_i32(s->pc - pc_start)); + tcg_const_i32(s->pc - s->base.pc_next)); tcg_gen_exit_tb(NULL, 0); s->base.is_jmp = DISAS_NORETURN; break; @@ -7738,7 +7744,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_vmmcall(cpu_env); break; @@ -7750,7 +7756,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1)); break; @@ -7762,7 +7768,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1)); break; @@ -7788,7 +7794,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_clgi(cpu_env); break; @@ -7934,7 +7940,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_jmp_im(s, s->base.pc_next - s->cs_base); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); } @@ -8386,7 +8392,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); break; } gen_lea_modrm(env, s, modrm); @@ -8399,7 +8405,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); break; } gen_lea_modrm(env, s, modrm); @@ -8411,7 +8417,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); break; } gen_lea_modrm(env, s, modrm); @@ -8424,7 +8430,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); break; } gen_helper_update_mxcsr(cpu_env); @@ -8633,7 +8639,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0x1c2: case 0x1c4 ... 0x1c6: case 0x1d0 ... 0x1fe: - gen_sse(env, s, b, pc_start); + gen_sse(env, s, b); break; default: goto unknown_op; From f66c8e8cd923a38e454c2d4731bccc0eede1a652 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:11 -0700 Subject: [PATCH 07/37] target/i386: Return bool from disas_insn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of returning the new pc, which is present in DisasContext, return true if an insn was translated. This is false when we detect a page crossing and must undo the insn under translation. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-3-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 44 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 16bf56dbc7..3f3e79c096 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -4707,7 +4707,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b) /* convert one instruction. s->base.is_jmp is set if the translation must be stopped. Return the next pc value */ -static target_ulong disas_insn(DisasContext *s, CPUState *cpu) +static bool disas_insn(DisasContext *s, CPUState *cpu) { CPUX86State *env = cpu->env_ptr; int b, prefixes; @@ -4734,15 +4734,16 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; case 1: gen_exception_gpf(s); - return s->pc; + return true; case 2: /* Restore state that may affect the next instruction. */ + s->pc = s->base.pc_next; s->cc_op_dirty = orig_cc_op_dirty; s->cc_op = orig_cc_op; s->base.num_insns--; tcg_remove_ops_after(s->prev_insn_end); s->base.is_jmp = DISAS_TOO_MANY; - return s->base.pc_next; + return false; default: g_assert_not_reached(); } @@ -8644,13 +8645,13 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) default: goto unknown_op; } - return s->pc; + return true; illegal_op: gen_illegal_opcode(s); - return s->pc; + return true; unknown_op: gen_unknown_opcode(env, s); - return s->pc; + return true; } void tcg_x86_init(void) @@ -8815,7 +8816,6 @@ static void i386_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu) static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); - target_ulong pc_next; #ifdef TARGET_VSYSCALL_PAGE /* @@ -8828,21 +8828,23 @@ static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) } #endif - pc_next = disas_insn(dc, cpu); - dc->base.pc_next = pc_next; + if (disas_insn(dc, cpu)) { + target_ulong pc_next = dc->pc; + dc->base.pc_next = pc_next; - if (dc->base.is_jmp == DISAS_NEXT) { - if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { - /* - * If single step mode, we generate only one instruction and - * generate an exception. - * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear - * the flag and abort the translation to give the irqs a - * chance to happen. - */ - dc->base.is_jmp = DISAS_TOO_MANY; - } else if (!is_same_page(&dc->base, pc_next)) { - dc->base.is_jmp = DISAS_TOO_MANY; + if (dc->base.is_jmp == DISAS_NEXT) { + if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { + /* + * If single step mode, we generate only one instruction and + * generate an exception. + * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear + * the flag and abort the translation to give the irqs a + * chance to happen. + */ + dc->base.is_jmp = DISAS_TOO_MANY; + } else if (!is_same_page(&dc->base, pc_next)) { + dc->base.is_jmp = DISAS_TOO_MANY; + } } } } From 522365508ed88f3c975faae6d4d0ec64f53e14f9 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:12 -0700 Subject: [PATCH 08/37] target/i386: Remove cur_eip argument to gen_exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All callers pass s->base.pc_next - s->cs_base, which we can just as well compute within the function. Note the special case of EXCP_VSYSCALL in which s->cs_base wasn't subtracted, but cs_base is always zero in 64-bit mode, when vsyscall is used. Reviewed-by: Paolo Bonzini Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-4-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 3f3e79c096..617832fcb0 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -1332,10 +1332,10 @@ static void gen_helper_fp_arith_STN_ST0(int op, int opreg) } } -static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip) +static void gen_exception(DisasContext *s, int trapno) { gen_update_cc_op(s); - gen_jmp_im(s, cur_eip); + gen_jmp_im(s, s->base.pc_next - s->cs_base); gen_helper_raise_exception(cpu_env, tcg_const_i32(trapno)); s->base.is_jmp = DISAS_NORETURN; } @@ -1344,13 +1344,13 @@ static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip) the instruction is known, but it isn't allowed in the current cpu mode. */ static void gen_illegal_opcode(DisasContext *s) { - gen_exception(s, EXCP06_ILLOP, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP06_ILLOP); } /* Generate #GP for the current instruction. */ static void gen_exception_gpf(DisasContext *s) { - gen_exception(s, EXCP0D_GPF, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP0D_GPF); } /* Check for cpl == 0; if not, raise #GP and return false. */ @@ -3267,7 +3267,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b) } /* simple MMX/SSE operation */ if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); return; } if (s->flags & HF_EM_MASK) { @@ -6077,7 +6077,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (s->flags & (HF_EM_MASK | HF_TS_MASK)) { /* if CR0.EM or CR0.TS are set, generate an FPU exception */ /* XXX: what to do if illegal op ? */ - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } modrm = x86_ldub_code(env, s); @@ -7302,7 +7302,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; val = x86_ldub_code(env, s); if (val == 0) { - gen_exception(s, EXCP00_DIVZ, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP00_DIVZ); } else { gen_helper_aam(cpu_env, tcg_const_i32(val)); set_cc_op(s, CC_OP_LOGICB); @@ -7336,7 +7336,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0x9b: /* fwait */ if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); } else { gen_helper_fwait(cpu_env); } @@ -8393,7 +8393,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_lea_modrm(env, s, modrm); @@ -8406,7 +8406,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_lea_modrm(env, s, modrm); @@ -8418,7 +8418,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_lea_modrm(env, s, modrm); @@ -8431,7 +8431,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, s->base.pc_next - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_helper_update_mxcsr(cpu_env); @@ -8822,7 +8822,7 @@ static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) * Detect entry into the vsyscall page and invoke the syscall. */ if ((dc->base.pc_next & TARGET_PAGE_MASK) == TARGET_VSYSCALL_PAGE) { - gen_exception(dc, EXCP_VSYSCALL, dc->base.pc_next); + gen_exception(dc, EXCP_VSYSCALL); dc->base.pc_next = dc->pc + 1; return; } From 8ed6c985016961625b6e1c188e93e817f398b339 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:13 -0700 Subject: [PATCH 09/37] target/i386: Remove cur_eip, next_eip arguments to gen_interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All callers pass s->base.pc_next and s->pc, which we can just as well compute within the function. Adjust to use tcg_constant_i32 while we're at it. Reviewed-by: Paolo Bonzini Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-5-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 617832fcb0..5a9c3b1e71 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -2627,13 +2627,12 @@ static void gen_unknown_opcode(CPUX86State *env, DisasContext *s) /* an interrupt is different from an exception because of the privilege checks */ -static void gen_interrupt(DisasContext *s, int intno, - target_ulong cur_eip, target_ulong next_eip) +static void gen_interrupt(DisasContext *s, int intno) { gen_update_cc_op(s); - gen_jmp_im(s, cur_eip); - gen_helper_raise_interrupt(cpu_env, tcg_const_i32(intno), - tcg_const_i32(next_eip - cur_eip)); + gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_helper_raise_interrupt(cpu_env, tcg_constant_i32(intno), + tcg_constant_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; } @@ -7342,12 +7341,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } break; case 0xcc: /* int3 */ - gen_interrupt(s, EXCP03_INT3, s->base.pc_next - s->cs_base, s->pc - s->cs_base); + gen_interrupt(s, EXCP03_INT3); break; case 0xcd: /* int N */ val = x86_ldub_code(env, s); if (check_vm86_iopl(s)) { - gen_interrupt(s, val, s->base.pc_next - s->cs_base, s->pc - s->cs_base); + gen_interrupt(s, val); } break; case 0xce: /* into */ From 65e4af230d76769fe5a68efaf2145924e128fc51 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:14 -0700 Subject: [PATCH 10/37] target/i386: Create gen_update_eip_cur Like gen_update_cc_op, sync EIP before doing something that could raise an exception. Replace all gen_jmp_im that use s->base.pc_next. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-6-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 52 ++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 5a9c3b1e71..85253e1e17 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -511,10 +511,14 @@ static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d) } } -static inline void gen_jmp_im(DisasContext *s, target_ulong pc) +static void gen_jmp_im(DisasContext *s, target_ulong pc) { - tcg_gen_movi_tl(s->tmp0, pc); - gen_op_jmp_v(s->tmp0); + gen_op_jmp_v(tcg_constant_tl(pc)); +} + +static void gen_update_eip_cur(DisasContext *s) +{ + gen_jmp_im(s, s->base.pc_next - s->cs_base); } /* Compute SEG:REG into A0. SEG is selected from the override segment @@ -703,7 +707,7 @@ static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port, target_ulong next_eip = s->pc - s->cs_base; gen_update_cc_op(s); - gen_jmp_im(s, cur_eip); + gen_update_eip_cur(s); if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { svm_flags |= SVM_IOIO_REP_MASK; } @@ -1335,7 +1339,7 @@ static void gen_helper_fp_arith_STN_ST0(int op, int opreg) static void gen_exception(DisasContext *s, int trapno) { gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_raise_exception(cpu_env, tcg_const_i32(trapno)); s->base.is_jmp = DISAS_NORETURN; } @@ -2630,7 +2634,7 @@ static void gen_unknown_opcode(CPUX86State *env, DisasContext *s) static void gen_interrupt(DisasContext *s, int intno) { gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_raise_interrupt(cpu_env, tcg_constant_i32(intno), tcg_constant_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; @@ -6831,7 +6835,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) do_lret: if (PE(s) && !VM86(s)) { gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_lret_protected(cpu_env, tcg_const_i32(dflag - 1), tcg_const_i32(val)); } else { @@ -7327,7 +7331,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (prefixes & PREFIX_REPZ) { gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_pause(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; } @@ -7353,7 +7357,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (CODE64(s)) goto illegal_op; gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_into(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); break; #ifdef WANT_ICEBP @@ -7460,7 +7464,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0x132: /* rdmsr */ if (check_cpl0(s)) { gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); if (b & 2) { gen_helper_rdmsr(cpu_env); } else { @@ -7472,7 +7476,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; case 0x131: /* rdtsc */ gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); } @@ -7483,7 +7487,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; case 0x133: /* rdpmc */ gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_rdpmc(cpu_env); s->base.is_jmp = DISAS_NORETURN; break; @@ -7513,7 +7517,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0x105: /* syscall */ /* XXX: is it usable in real mode ? */ gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_syscall(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); /* TF handling for the syscall insn is different. The TF bit is checked after the syscall insn completes. This allows #DB to not be @@ -7539,13 +7543,13 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) #endif case 0x1a2: /* cpuid */ gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_cpuid(cpu_env); break; case 0xf4: /* hlt */ if (check_cpl0(s)) { gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_hlt(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; } @@ -7642,7 +7646,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); tcg_gen_mov_tl(s->A0, cpu_regs[R_EAX]); gen_extu(s->aflag, s->A0); gen_add_A0_ds_seg(s); @@ -7654,7 +7658,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); s->base.is_jmp = DISAS_NORETURN; break; @@ -7732,7 +7736,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1), tcg_const_i32(s->pc - s->base.pc_next)); tcg_gen_exit_tb(NULL, 0); @@ -7744,7 +7748,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmmcall(cpu_env); break; @@ -7756,7 +7760,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1)); break; @@ -7768,7 +7772,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1)); break; @@ -7794,7 +7798,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); gen_helper_clgi(cpu_env); break; @@ -7940,7 +7944,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, s->base.pc_next - s->cs_base); + gen_update_eip_cur(s); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); } @@ -8853,7 +8857,7 @@ static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) DisasContext *dc = container_of(dcbase, DisasContext, base); if (dc->base.is_jmp == DISAS_TOO_MANY) { - gen_jmp_im(dc, dc->base.pc_next - dc->cs_base); + gen_update_eip_cur(dc); gen_eob(dc); } } From 09e99df4d54225619d57cecd9d3c61c719236a24 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:15 -0700 Subject: [PATCH 11/37] target/i386: Create gen_update_eip_next Sync EIP before exiting a translation block. Replace all gen_jmp_im that use s->pc. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-7-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 45 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 85253e1e17..4c1548da8e 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -521,6 +521,11 @@ static void gen_update_eip_cur(DisasContext *s) gen_jmp_im(s, s->base.pc_next - s->cs_base); } +static void gen_update_eip_next(DisasContext *s) +{ + gen_jmp_im(s, s->pc - s->cs_base); +} + /* Compute SEG:REG into A0. SEG is selected from the override segment (OVR_SEG) and the default segment (DEF_SEG). OVR_SEG may be -1 to indicate no override. */ @@ -5719,7 +5724,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_pop_update(s, ot); /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp. */ if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); if (reg == R_SS) { s->flags &= ~HF_TF_MASK; gen_eob_inhibit_irq(s, true); @@ -5734,7 +5739,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_movl_seg_T0(s, (b >> 3) & 7); gen_pop_update(s, ot); if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } break; @@ -5785,7 +5790,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_movl_seg_T0(s, reg); /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp. */ if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); if (reg == R_SS) { s->flags &= ~HF_TF_MASK; gen_eob_inhibit_irq(s, true); @@ -5983,7 +5988,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) /* then put the data */ gen_op_mov_reg_v(s, ot, reg, s->T1); if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } break; @@ -7039,7 +7044,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_pop_update(s, ot); set_cc_op(s, CC_OP_EFLAGS); /* abort translation because TF/AC flag may change */ - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } break; @@ -7375,7 +7380,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (check_iopl(s)) { gen_helper_sti(cpu_env); /* interruptions are enabled only the first insn after sti */ - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob_inhibit_irq(s, true); } break; @@ -7451,7 +7456,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_set_label(l3); - gen_jmp_im(s, next_eip); + gen_update_eip_next(s); tcg_gen_br(l2); gen_set_label(l1); @@ -7469,7 +7474,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_helper_rdmsr(cpu_env); } else { gen_helper_wrmsr(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } } @@ -7669,7 +7674,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_helper_clac(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -7679,7 +7684,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_helper_stac(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -7724,7 +7729,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_trunc_tl_i32(s->tmp2_i32, cpu_regs[R_ECX]); gen_helper_xsetbv(cpu_env, s->tmp2_i32, s->tmp1_i64); /* End TB because translation flags may change. */ - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -7786,7 +7791,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_update_cc_op(s); gen_helper_stgi(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -7825,7 +7830,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_ext32u_tl(s->A0, cpu_regs[R_EAX]); } gen_helper_flush_page(cpu_env, s->A0); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -7909,7 +7914,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_andi_tl(s->T1, s->T1, ~0xe); tcg_gen_or_tl(s->T0, s->T0, s->T1); gen_helper_write_crN(cpu_env, tcg_constant_i32(0), s->T0); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -7920,7 +7925,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_INVLPG); gen_lea_modrm(env, s, modrm); gen_helper_flush_page(cpu_env, s->A0); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -8320,7 +8325,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0 + reg); gen_op_mov_v_reg(s, ot, s->T0, rm); gen_helper_write_crN(cpu_env, tcg_constant_i32(reg), s->T0); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } else { gen_svm_check_intercept(s, SVM_EXIT_READ_CR0 + reg); @@ -8355,7 +8360,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_op_mov_v_reg(s, ot, s->T0, rm); tcg_gen_movi_i32(s->tmp2_i32, reg); gen_helper_set_dr(cpu_env, s->tmp2_i32, s->T0); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } else { gen_svm_check_intercept(s, SVM_EXIT_READ_DR0 + reg); @@ -8370,7 +8375,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0); gen_helper_clts(cpu_env); /* abort block because static cpu state changed */ - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); } break; @@ -8468,7 +8473,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) /* XRSTOR is how MPX is enabled, which changes how we translate. Thus we need to end the TB. */ gen_update_cc_op(s); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob(s); break; @@ -8602,7 +8607,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) g_assert_not_reached(); #else gen_update_cc_op(s); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_helper_rsm(cpu_env); #endif /* CONFIG_USER_ONLY */ gen_eob(s); From 200ef603996bda2cd1e3c5ae63311bfb7d229e3b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:16 -0700 Subject: [PATCH 12/37] target/i386: Introduce DISAS_EOB* Add a few DISAS_TARGET_* aliases to reduce the number of calls to gen_eob() and gen_eob_inhibit_irq(). So far, only update i386_tr_translate_insn for exiting the block because of single-step or previous inhibit irq. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-8-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 4c1548da8e..caa22af5a7 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -132,6 +132,10 @@ typedef struct DisasContext { TCGOp *prev_insn_end; } DisasContext; +#define DISAS_EOB_ONLY DISAS_TARGET_0 +#define DISAS_EOB_NEXT DISAS_TARGET_1 +#define DISAS_EOB_INHIBIT_IRQ DISAS_TARGET_2 + /* The environment in which user-only runs is constrained. */ #ifdef CONFIG_USER_ONLY #define PE(S) true @@ -8849,7 +8853,7 @@ static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) * the flag and abort the translation to give the irqs a * chance to happen. */ - dc->base.is_jmp = DISAS_TOO_MANY; + dc->base.is_jmp = DISAS_EOB_NEXT; } else if (!is_same_page(&dc->base, pc_next)) { dc->base.is_jmp = DISAS_TOO_MANY; } @@ -8861,9 +8865,24 @@ static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); - if (dc->base.is_jmp == DISAS_TOO_MANY) { + switch (dc->base.is_jmp) { + case DISAS_NORETURN: + break; + case DISAS_TOO_MANY: + case DISAS_EOB_NEXT: + gen_update_cc_op(dc); gen_update_eip_cur(dc); + /* fall through */ + case DISAS_EOB_ONLY: gen_eob(dc); + break; + case DISAS_EOB_INHIBIT_IRQ: + gen_update_cc_op(dc); + gen_update_eip_cur(dc); + gen_eob_inhibit_irq(dc, true); + break; + default: + g_assert_not_reached(); } } From 4da4523c6ce78299b1a3966ec285441d190562a5 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:17 -0700 Subject: [PATCH 13/37] target/i386: Use DISAS_EOB* in gen_movl_seg_T0 Set is_jmp properly in gen_movl_seg_T0, so that the callers need to nothing special. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-9-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 36 +++++------------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index caa22af5a7..8c0ef0f212 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -2457,13 +2457,15 @@ static void gen_movl_seg_T0(DisasContext *s, X86Seg seg_reg) because ss32 may change. For R_SS, translation must always stop as a special handling must be done to disable hardware interrupts for the next instruction */ - if (seg_reg == R_SS || (CODE32(s) && seg_reg < R_FS)) { - s->base.is_jmp = DISAS_TOO_MANY; + if (seg_reg == R_SS) { + s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; + } else if (CODE32(s) && seg_reg < R_FS) { + s->base.is_jmp = DISAS_EOB_NEXT; } } else { gen_op_movl_seg_T0_vm(s, seg_reg); if (seg_reg == R_SS) { - s->base.is_jmp = DISAS_TOO_MANY; + s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; } } } @@ -5726,26 +5728,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) ot = gen_pop_T0(s); gen_movl_seg_T0(s, reg); gen_pop_update(s, ot); - /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp. */ - if (s->base.is_jmp) { - gen_update_eip_next(s); - if (reg == R_SS) { - s->flags &= ~HF_TF_MASK; - gen_eob_inhibit_irq(s, true); - } else { - gen_eob(s); - } - } break; case 0x1a1: /* pop fs */ case 0x1a9: /* pop gs */ ot = gen_pop_T0(s); gen_movl_seg_T0(s, (b >> 3) & 7); gen_pop_update(s, ot); - if (s->base.is_jmp) { - gen_update_eip_next(s); - gen_eob(s); - } break; /**************************/ @@ -5792,16 +5780,6 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0); gen_movl_seg_T0(s, reg); - /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp. */ - if (s->base.is_jmp) { - gen_update_eip_next(s); - if (reg == R_SS) { - s->flags &= ~HF_TF_MASK; - gen_eob_inhibit_irq(s, true); - } else { - gen_eob(s); - } - } break; case 0x8c: /* mov Gv, seg */ modrm = x86_ldub_code(env, s); @@ -5991,10 +5969,6 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_movl_seg_T0(s, op); /* then put the data */ gen_op_mov_reg_v(s, ot, reg, s->T1); - if (s->base.is_jmp) { - gen_update_eip_next(s); - gen_eob(s); - } break; /************************/ From 634a405193cc7a828e59c189ea805a8a51a6a959 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:18 -0700 Subject: [PATCH 14/37] target/i386: Use DISAS_EOB_NEXT Replace sequences of gen_update_cc_op, gen_update_eip_next, and gen_eob with the new is_jmp enumerator. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-10-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 40 ++++++++++++------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 8c0ef0f212..717c978381 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -7022,8 +7022,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_pop_update(s, ot); set_cc_op(s, CC_OP_EFLAGS); /* abort translation because TF/AC flag may change */ - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } break; case 0x9e: /* sahf */ @@ -7452,8 +7451,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_helper_rdmsr(cpu_env); } else { gen_helper_wrmsr(cpu_env); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } } break; @@ -7652,8 +7650,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_helper_clac(cpu_env); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xcb: /* stac */ @@ -7662,8 +7659,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_helper_stac(cpu_env); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(1): /* sidt */ @@ -7707,8 +7703,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_trunc_tl_i32(s->tmp2_i32, cpu_regs[R_ECX]); gen_helper_xsetbv(cpu_env, s->tmp2_i32, s->tmp1_i64); /* End TB because translation flags may change. */ - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xd8: /* VMRUN */ @@ -7769,8 +7764,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_update_cc_op(s); gen_helper_stgi(cpu_env); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xdd: /* CLGI */ @@ -7808,8 +7802,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_ext32u_tl(s->A0, cpu_regs[R_EAX]); } gen_helper_flush_page(cpu_env, s->A0); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(2): /* lgdt */ @@ -7892,8 +7885,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_andi_tl(s->T1, s->T1, ~0xe); tcg_gen_or_tl(s->T0, s->T0, s->T1); gen_helper_write_crN(cpu_env, tcg_constant_i32(0), s->T0); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(7): /* invlpg */ @@ -7903,8 +7895,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_INVLPG); gen_lea_modrm(env, s, modrm); gen_helper_flush_page(cpu_env, s->A0); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xf8: /* swapgs */ @@ -8303,8 +8294,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0 + reg); gen_op_mov_v_reg(s, ot, s->T0, rm); gen_helper_write_crN(cpu_env, tcg_constant_i32(reg), s->T0); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } else { gen_svm_check_intercept(s, SVM_EXIT_READ_CR0 + reg); gen_helper_read_crN(s->T0, cpu_env, tcg_constant_i32(reg)); @@ -8338,8 +8328,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_op_mov_v_reg(s, ot, s->T0, rm); tcg_gen_movi_i32(s->tmp2_i32, reg); gen_helper_set_dr(cpu_env, s->tmp2_i32, s->T0); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } else { gen_svm_check_intercept(s, SVM_EXIT_READ_DR0 + reg); tcg_gen_movi_i32(s->tmp2_i32, reg); @@ -8353,8 +8342,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0); gen_helper_clts(cpu_env); /* abort block because static cpu state changed */ - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } break; /* MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4 support */ @@ -8450,9 +8438,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_helper_xrstor(cpu_env, s->A0, s->tmp1_i64); /* XRSTOR is how MPX is enabled, which changes how we translate. Thus we need to end the TB. */ - gen_update_cc_op(s); - gen_update_eip_next(s); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(6): /* xsaveopt / clwb */ From 6424ac8eec7d5e20a0ed6be1031108ca167299e4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:19 -0700 Subject: [PATCH 15/37] target/i386: USe DISAS_EOB_ONLY Replace lone calls to gen_eob() with the new enumerator. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-11-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 717c978381..6b16c0b62c 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -6835,7 +6835,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) /* add stack offset */ gen_stack_update(s, val + (2 << dflag)); } - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; break; case 0xcb: /* lret */ val = 0; @@ -6853,7 +6853,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_const_i32(s->pc - s->cs_base)); } set_cc_op(s, CC_OP_EFLAGS); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; break; case 0xe8: /* call im */ { @@ -7439,7 +7439,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_set_label(l1); gen_jmp_im(s, tval); gen_set_label(l2); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; } break; case 0x130: /* wrmsr */ @@ -7480,7 +7480,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_exception_gpf(s); } else { gen_helper_sysenter(cpu_env); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; } break; case 0x135: /* sysexit */ @@ -7491,7 +7491,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_exception_gpf(s); } else { gen_helper_sysexit(cpu_env, tcg_const_i32(dflag - 1)); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; } break; #ifdef TARGET_X86_64 @@ -8574,7 +8574,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_update_eip_next(s); gen_helper_rsm(cpu_env); #endif /* CONFIG_USER_ONLY */ - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; break; case 0x1b8: /* SSE4.2 popcnt */ if ((prefixes & (PREFIX_REPZ | PREFIX_LOCK | PREFIX_REPNZ)) != From ad1d6f072d02e99114ea28d674131459b9c70897 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:20 -0700 Subject: [PATCH 16/37] target/i386: Create cur_insn_len, cur_insn_len_i32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create common routines for computing the length of the insn. Use tcg_constant_i32 in the new function, while we're at it. Reviewed-by: Paolo Bonzini Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-12-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 6b16c0b62c..fe99c4361c 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -530,6 +530,16 @@ static void gen_update_eip_next(DisasContext *s) gen_jmp_im(s, s->pc - s->cs_base); } +static int cur_insn_len(DisasContext *s) +{ + return s->pc - s->base.pc_next; +} + +static TCGv_i32 cur_insn_len_i32(DisasContext *s) +{ + return tcg_constant_i32(cur_insn_len(s)); +} + /* Compute SEG:REG into A0. SEG is selected from the override segment (OVR_SEG) and the default segment (DEF_SEG). OVR_SEG may be -1 to indicate no override. */ @@ -712,9 +722,6 @@ static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port, gen_helper_check_io(cpu_env, port, tcg_constant_i32(1 << ot)); } if (GUEST(s)) { - target_ulong cur_eip = s->base.pc_next - s->cs_base; - target_ulong next_eip = s->pc - s->cs_base; - gen_update_cc_op(s); gen_update_eip_cur(s); if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { @@ -723,7 +730,7 @@ static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port, svm_flags |= 1 << (SVM_IOIO_SIZE_SHIFT + ot); gen_helper_svm_check_io(cpu_env, port, tcg_constant_i32(svm_flags), - tcg_constant_i32(next_eip - cur_eip)); + cur_insn_len_i32(s)); } return true; #endif @@ -2028,7 +2035,7 @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes) } s->pc += num_bytes; - if (unlikely(s->pc - s->base.pc_next > X86_MAX_INSN_LENGTH)) { + if (unlikely(cur_insn_len(s) > X86_MAX_INSN_LENGTH)) { /* If the instruction's 16th byte is on a different page than the 1st, a * page fault on the second page wins over the general protection fault * caused by the instruction being too long. @@ -2647,7 +2654,7 @@ static void gen_interrupt(DisasContext *s, int intno) gen_update_cc_op(s); gen_update_eip_cur(s); gen_helper_raise_interrupt(cpu_env, tcg_constant_i32(intno), - tcg_constant_i32(s->pc - s->base.pc_next)); + cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; } @@ -7314,7 +7321,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (prefixes & PREFIX_REPZ) { gen_update_cc_op(s); gen_update_eip_cur(s); - gen_helper_pause(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); + gen_helper_pause(cpu_env, cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; } break; @@ -7340,7 +7347,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; gen_update_cc_op(s); gen_update_eip_cur(s); - gen_helper_into(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); + gen_helper_into(cpu_env, cur_insn_len_i32(s)); break; #ifdef WANT_ICEBP case 0xf1: /* icebp (undocumented, exits to external debugger) */ @@ -7499,7 +7506,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) /* XXX: is it usable in real mode ? */ gen_update_cc_op(s); gen_update_eip_cur(s); - gen_helper_syscall(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); + gen_helper_syscall(cpu_env, cur_insn_len_i32(s)); /* TF handling for the syscall insn is different. The TF bit is checked after the syscall insn completes. This allows #DB to not be generated after one has entered CPL0 if TF is set in FMASK. */ @@ -7531,7 +7538,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (check_cpl0(s)) { gen_update_cc_op(s); gen_update_eip_cur(s); - gen_helper_hlt(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); + gen_helper_hlt(cpu_env, cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; } break; @@ -7640,7 +7647,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_update_cc_op(s); gen_update_eip_cur(s); - gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - s->base.pc_next)); + gen_helper_mwait(cpu_env, cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; break; @@ -7716,7 +7723,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_update_cc_op(s); gen_update_eip_cur(s); gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1), - tcg_const_i32(s->pc - s->base.pc_next)); + cur_insn_len_i32(s)); tcg_gen_exit_tb(NULL, 0); s->base.is_jmp = DISAS_NORETURN; break; From 122e6d7b4a51755eeabab15570a152c8379f652d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:21 -0700 Subject: [PATCH 17/37] target/i386: Remove cur_eip, next_eip arguments to gen_repz* All callers pass s->base.pc_next and s->pc, which we can just as well compute within the functions. Pull out common helpers and reduce the amount of code under macros. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-13-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 116 ++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index fe99c4361c..c8ef9f0356 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -736,7 +736,7 @@ static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port, #endif } -static inline void gen_movs(DisasContext *s, MemOp ot) +static void gen_movs(DisasContext *s, MemOp ot) { gen_string_movl_A0_ESI(s); gen_op_ld_v(s, ot, s->T0, s->A0); @@ -1156,18 +1156,18 @@ static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1) /* XXX: does not work with gdbstub "ice" single step - not a serious problem */ -static TCGLabel *gen_jz_ecx_string(DisasContext *s, target_ulong next_eip) +static TCGLabel *gen_jz_ecx_string(DisasContext *s) { TCGLabel *l1 = gen_new_label(); TCGLabel *l2 = gen_new_label(); gen_op_jnz_ecx(s, s->aflag, l1); gen_set_label(l2); - gen_jmp_tb(s, next_eip, 1); + gen_jmp_tb(s, s->pc - s->cs_base, 1); gen_set_label(l1); return l2; } -static inline void gen_stos(DisasContext *s, MemOp ot) +static void gen_stos(DisasContext *s, MemOp ot) { gen_op_mov_v_reg(s, MO_32, s->T0, R_EAX); gen_string_movl_A0_EDI(s); @@ -1176,7 +1176,7 @@ static inline void gen_stos(DisasContext *s, MemOp ot) gen_op_add_reg_T0(s, s->aflag, R_EDI); } -static inline void gen_lods(DisasContext *s, MemOp ot) +static void gen_lods(DisasContext *s, MemOp ot) { gen_string_movl_A0_ESI(s); gen_op_ld_v(s, ot, s->T0, s->A0); @@ -1185,7 +1185,7 @@ static inline void gen_lods(DisasContext *s, MemOp ot) gen_op_add_reg_T0(s, s->aflag, R_ESI); } -static inline void gen_scas(DisasContext *s, MemOp ot) +static void gen_scas(DisasContext *s, MemOp ot) { gen_string_movl_A0_EDI(s); gen_op_ld_v(s, ot, s->T1, s->A0); @@ -1194,7 +1194,7 @@ static inline void gen_scas(DisasContext *s, MemOp ot) gen_op_add_reg_T0(s, s->aflag, R_EDI); } -static inline void gen_cmps(DisasContext *s, MemOp ot) +static void gen_cmps(DisasContext *s, MemOp ot) { gen_string_movl_A0_EDI(s); gen_op_ld_v(s, ot, s->T1, s->A0); @@ -1222,7 +1222,7 @@ static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot) } } -static inline void gen_ins(DisasContext *s, MemOp ot) +static void gen_ins(DisasContext *s, MemOp ot) { gen_string_movl_A0_EDI(s); /* Note: we must do this dummy write first to be restartable in @@ -1238,7 +1238,7 @@ static inline void gen_ins(DisasContext *s, MemOp ot) gen_bpt_io(s, s->tmp2_i32, ot); } -static inline void gen_outs(DisasContext *s, MemOp ot) +static void gen_outs(DisasContext *s, MemOp ot) { gen_string_movl_A0_ESI(s); gen_op_ld_v(s, ot, s->T0, s->A0); @@ -1252,42 +1252,49 @@ static inline void gen_outs(DisasContext *s, MemOp ot) gen_bpt_io(s, s->tmp2_i32, ot); } -/* same method as Valgrind : we generate jumps to current or next - instruction */ -#define GEN_REPZ(op) \ -static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, \ - target_ulong cur_eip, target_ulong next_eip) \ -{ \ - TCGLabel *l2; \ - gen_update_cc_op(s); \ - l2 = gen_jz_ecx_string(s, next_eip); \ - gen_ ## op(s, ot); \ - gen_op_add_reg_im(s, s->aflag, R_ECX, -1); \ - /* a loop would cause two single step exceptions if ECX = 1 \ - before rep string_insn */ \ - if (s->repz_opt) \ - gen_op_jz_ecx(s, s->aflag, l2); \ - gen_jmp(s, cur_eip); \ +/* Generate jumps to current or next instruction */ +static void gen_repz(DisasContext *s, MemOp ot, + void (*fn)(DisasContext *s, MemOp ot)) +{ + TCGLabel *l2; + gen_update_cc_op(s); + l2 = gen_jz_ecx_string(s); + fn(s, ot); + gen_op_add_reg_im(s, s->aflag, R_ECX, -1); + /* + * A loop would cause two single step exceptions if ECX = 1 + * before rep string_insn + */ + if (s->repz_opt) { + gen_op_jz_ecx(s, s->aflag, l2); + } + gen_jmp(s, s->base.pc_next - s->cs_base); } -#define GEN_REPZ2(op) \ -static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, \ - target_ulong cur_eip, \ - target_ulong next_eip, \ - int nz) \ -{ \ - TCGLabel *l2; \ - gen_update_cc_op(s); \ - l2 = gen_jz_ecx_string(s, next_eip); \ - gen_ ## op(s, ot); \ - gen_op_add_reg_im(s, s->aflag, R_ECX, -1); \ - gen_update_cc_op(s); \ - gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); \ - if (s->repz_opt) \ - gen_op_jz_ecx(s, s->aflag, l2); \ - gen_jmp(s, cur_eip); \ +#define GEN_REPZ(op) \ + static inline void gen_repz_ ## op(DisasContext *s, MemOp ot) \ + { gen_repz(s, ot, gen_##op); } + +static void gen_repz2(DisasContext *s, MemOp ot, int nz, + void (*fn)(DisasContext *s, MemOp ot)) +{ + TCGLabel *l2; + gen_update_cc_op(s); + l2 = gen_jz_ecx_string(s); + fn(s, ot); + gen_op_add_reg_im(s, s->aflag, R_ECX, -1); + gen_update_cc_op(s); + gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); + if (s->repz_opt) { + gen_op_jz_ecx(s, s->aflag, l2); + } + gen_jmp(s, s->base.pc_next - s->cs_base); } +#define GEN_REPZ2(op) \ + static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, int nz) \ + { gen_repz2(s, ot, nz, gen_##op); } + GEN_REPZ(movs) GEN_REPZ(stos) GEN_REPZ(lods) @@ -6623,8 +6630,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0xa5: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_movs(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base); + gen_repz_movs(s, ot); } else { gen_movs(s, ot); } @@ -6634,8 +6640,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0xab: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_stos(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base); + gen_repz_stos(s, ot); } else { gen_stos(s, ot); } @@ -6644,8 +6649,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0xad: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_lods(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base); + gen_repz_lods(s, ot); } else { gen_lods(s, ot); } @@ -6654,11 +6658,9 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0xaf: ot = mo_b_d(b, dflag); if (prefixes & PREFIX_REPNZ) { - gen_repz_scas(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base, 1); + gen_repz_scas(s, ot, 1); } else if (prefixes & PREFIX_REPZ) { - gen_repz_scas(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base, 0); + gen_repz_scas(s, ot, 0); } else { gen_scas(s, ot); } @@ -6668,11 +6670,9 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0xa7: ot = mo_b_d(b, dflag); if (prefixes & PREFIX_REPNZ) { - gen_repz_cmps(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base, 1); + gen_repz_cmps(s, ot, 1); } else if (prefixes & PREFIX_REPZ) { - gen_repz_cmps(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base, 0); + gen_repz_cmps(s, ot, 0); } else { gen_cmps(s, ot); } @@ -6690,8 +6690,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_io_start(); } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_ins(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base); + gen_repz_ins(s, ot); /* jump generated by gen_repz_ins */ } else { gen_ins(s, ot); @@ -6712,8 +6711,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_io_start(); } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_outs(s, ot, s->base.pc_next - s->cs_base, - s->pc - s->cs_base); + gen_repz_outs(s, ot); /* jump generated by gen_repz_outs */ } else { gen_outs(s, ot); From faf9ea5fa5558782b4031be16145758caf5e20e5 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:22 -0700 Subject: [PATCH 18/37] target/i386: Introduce DISAS_JUMP Drop the unused dest argument to gen_jr(). Remove most of the calls to gen_jr, and use DISAS_JUMP. Remove some unused loads of eip for lcall and ljmp. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-14-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index c8ef9f0356..7db6f617a1 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -135,6 +135,7 @@ typedef struct DisasContext { #define DISAS_EOB_ONLY DISAS_TARGET_0 #define DISAS_EOB_NEXT DISAS_TARGET_1 #define DISAS_EOB_INHIBIT_IRQ DISAS_TARGET_2 +#define DISAS_JUMP DISAS_TARGET_3 /* The environment in which user-only runs is constrained. */ #ifdef CONFIG_USER_ONLY @@ -222,7 +223,7 @@ STUB_HELPER(wrmsr, TCGv_env env) #endif static void gen_eob(DisasContext *s); -static void gen_jr(DisasContext *s, TCGv dest); +static void gen_jr(DisasContext *s); static void gen_jmp(DisasContext *s, target_ulong eip); static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num); static void gen_op(DisasContext *s1, int op, MemOp ot, int d); @@ -2385,7 +2386,7 @@ static void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) } else { /* jump to another page */ gen_jmp_im(s, eip); - gen_jr(s, s->tmp0); + gen_jr(s); } } @@ -2754,7 +2755,7 @@ static void gen_eob(DisasContext *s) } /* Jump to register */ -static void gen_jr(DisasContext *s, TCGv dest) +static void gen_jr(DisasContext *s) { do_gen_eob_worker(s, false, false, true); } @@ -5328,7 +5329,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_push_v(s, s->T1); gen_op_jmp_v(s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 3: /* lcall Ev */ if (mod == 3) { @@ -5349,8 +5350,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_const_i32(dflag - 1), tcg_const_i32(s->pc - s->cs_base)); } - tcg_gen_ld_tl(s->tmp4, cpu_env, offsetof(CPUX86State, eip)); - gen_jr(s, s->tmp4); + s->base.is_jmp = DISAS_JUMP; break; case 4: /* jmp Ev */ if (dflag == MO_16) { @@ -5358,7 +5358,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_op_jmp_v(s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 5: /* ljmp Ev */ if (mod == 3) { @@ -5376,8 +5376,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_op_movl_seg_T0_vm(s, R_CS); gen_op_jmp_v(s->T1); } - tcg_gen_ld_tl(s->tmp4, cpu_env, offsetof(CPUX86State, eip)); - gen_jr(s, s->tmp4); + s->base.is_jmp = DISAS_JUMP; break; case 6: /* push Ev */ gen_push_v(s, s->T0); @@ -6808,7 +6807,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) /* Note that gen_pop_T0 uses a zero-extending load. */ gen_op_jmp_v(s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 0xc3: /* ret */ ot = gen_pop_T0(s); @@ -6816,7 +6815,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) /* Note that gen_pop_T0 uses a zero-extending load. */ gen_op_jmp_v(s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 0xca: /* lret im */ val = x86_ldsw_code(env, s); @@ -8846,6 +8845,9 @@ static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) gen_update_eip_cur(dc); gen_eob_inhibit_irq(dc, true); break; + case DISAS_JUMP: + gen_jr(dc); + break; default: g_assert_not_reached(); } From 8c03ab9f74eb22d85e698f690e7035c1c7874cdf Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:23 -0700 Subject: [PATCH 19/37] target/i386: Truncate values for lcall_real to i32 Use i32 not int or tl for eip and cs arguments. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-15-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/helper.h | 2 +- target/i386/tcg/seg_helper.c | 6 ++---- target/i386/tcg/translate.c | 3 ++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/target/i386/helper.h b/target/i386/helper.h index ac3b4d1ee3..39a3c24182 100644 --- a/target/i386/helper.h +++ b/target/i386/helper.h @@ -37,7 +37,7 @@ DEF_HELPER_2(lldt, void, env, int) DEF_HELPER_2(ltr, void, env, int) DEF_HELPER_3(load_seg, void, env, int, int) DEF_HELPER_4(ljmp_protected, void, env, int, tl, tl) -DEF_HELPER_5(lcall_real, void, env, int, tl, int, int) +DEF_HELPER_5(lcall_real, void, env, i32, i32, int, i32) DEF_HELPER_5(lcall_protected, void, env, int, tl, int, tl) DEF_HELPER_2(iret_real, void, env, int) DEF_HELPER_3(iret_protected, void, env, int, int) diff --git a/target/i386/tcg/seg_helper.c b/target/i386/tcg/seg_helper.c index bffd82923f..539189b4d1 100644 --- a/target/i386/tcg/seg_helper.c +++ b/target/i386/tcg/seg_helper.c @@ -1504,14 +1504,12 @@ void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip, } /* real mode call */ -void helper_lcall_real(CPUX86State *env, int new_cs, target_ulong new_eip1, - int shift, int next_eip) +void helper_lcall_real(CPUX86State *env, uint32_t new_cs, uint32_t new_eip, + int shift, uint32_t next_eip) { - int new_eip; uint32_t esp, esp_mask; target_ulong ssp; - new_eip = new_eip1; esp = env->regs[R_ESP]; esp_mask = get_sp_mask(env->segs[R_SS].flags); ssp = env->segs[R_SS].base; diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 7db6f617a1..1aa5b37ea6 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -5346,7 +5346,8 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_const_tl(s->pc - s->cs_base)); } else { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); - gen_helper_lcall_real(cpu_env, s->tmp2_i32, s->T1, + tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); + gen_helper_lcall_real(cpu_env, s->tmp2_i32, s->tmp3_i32, tcg_const_i32(dflag - 1), tcg_const_i32(s->pc - s->cs_base)); } From 9e599bf70772808d3bd115902260cb924f88c939 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:24 -0700 Subject: [PATCH 20/37] target/i386: Create eip_next_* Create helpers for loading the address of the next insn. Use tcg_constant_* in adjacent code where convenient. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-16-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 55 +++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 1aa5b37ea6..be29ea7a03 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -541,6 +541,27 @@ static TCGv_i32 cur_insn_len_i32(DisasContext *s) return tcg_constant_i32(cur_insn_len(s)); } +static TCGv_i32 eip_next_i32(DisasContext *s) +{ + /* + * This function has two users: lcall_real (always 16-bit mode), and + * iret_protected (16, 32, or 64-bit mode). IRET only uses the value + * when EFLAGS.NT is set, which is illegal in 64-bit mode, which is + * why passing a 32-bit value isn't broken. To avoid using this where + * we shouldn't, return -1 in 64-bit mode so that execution goes into + * the weeds quickly. + */ + if (CODE64(s)) { + return tcg_constant_i32(-1); + } + return tcg_constant_i32(s->pc - s->cs_base); +} + +static TCGv eip_next_tl(DisasContext *s) +{ + return tcg_constant_tl(s->pc - s->cs_base); +} + /* Compute SEG:REG into A0. SEG is selected from the override segment (OVR_SEG) and the default segment (DEF_SEG). OVR_SEG may be -1 to indicate no override. */ @@ -1213,12 +1234,9 @@ static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot) /* user-mode cpu should not be in IOBPT mode */ g_assert_not_reached(); #else - TCGv_i32 t_size = tcg_const_i32(1 << ot); - TCGv t_next = tcg_const_tl(s->pc - s->cs_base); - + TCGv_i32 t_size = tcg_constant_i32(1 << ot); + TCGv t_next = eip_next_tl(s); gen_helper_bpt_io(cpu_env, t_port, t_size, t_next); - tcg_temp_free_i32(t_size); - tcg_temp_free(t_next); #endif /* CONFIG_USER_ONLY */ } } @@ -5324,9 +5342,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (dflag == MO_16) { tcg_gen_ext16u_tl(s->T0, s->T0); } - next_eip = s->pc - s->cs_base; - tcg_gen_movi_tl(s->T1, next_eip); - gen_push_v(s, s->T1); + gen_push_v(s, eip_next_tl(s)); gen_op_jmp_v(s->T0); gen_bnd_jmp(s); s->base.is_jmp = DISAS_JUMP; @@ -5342,14 +5358,14 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (PE(s) && !VM86(s)) { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); gen_helper_lcall_protected(cpu_env, s->tmp2_i32, s->T1, - tcg_const_i32(dflag - 1), - tcg_const_tl(s->pc - s->cs_base)); + tcg_constant_i32(dflag - 1), + eip_next_tl(s)); } else { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_lcall_real(cpu_env, s->tmp2_i32, s->tmp3_i32, - tcg_const_i32(dflag - 1), - tcg_const_i32(s->pc - s->cs_base)); + tcg_constant_i32(dflag - 1), + eip_next_i32(s)); } s->base.is_jmp = DISAS_JUMP; break; @@ -5372,7 +5388,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (PE(s) && !VM86(s)) { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); gen_helper_ljmp_protected(cpu_env, s->tmp2_i32, s->T1, - tcg_const_tl(s->pc - s->cs_base)); + eip_next_tl(s)); } else { gen_op_movl_seg_T0_vm(s, R_CS); gen_op_jmp_v(s->T1); @@ -6854,8 +6870,8 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } gen_helper_iret_real(cpu_env, tcg_const_i32(dflag - 1)); } else { - gen_helper_iret_protected(cpu_env, tcg_const_i32(dflag - 1), - tcg_const_i32(s->pc - s->cs_base)); + gen_helper_iret_protected(cpu_env, tcg_constant_i32(dflag - 1), + eip_next_i32(s)); } set_cc_op(s, CC_OP_EFLAGS); s->base.is_jmp = DISAS_EOB_ONLY; @@ -6867,15 +6883,13 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } else { tval = (int16_t)insn_get(env, s, MO_16); } - next_eip = s->pc - s->cs_base; - tval += next_eip; + tval += s->pc - s->cs_base; if (dflag == MO_16) { tval &= 0xffff; } else if (!CODE64(s)) { tval &= 0xffffffff; } - tcg_gen_movi_tl(s->T0, next_eip); - gen_push_v(s, s->T0); + gen_push_v(s, eip_next_tl(s)); gen_bnd_jmp(s); gen_jmp(s, tval); } @@ -7409,8 +7423,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) TCGLabel *l1, *l2, *l3; tval = (int8_t)insn_get(env, s, MO_8); - next_eip = s->pc - s->cs_base; - tval += next_eip; + tval += s->pc - s->cs_base; if (dflag == MO_16) { tval &= 0xffff; } From 202005f1f8e0b8a5a3a4b1527c4f484a4be2d5a6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:25 -0700 Subject: [PATCH 21/37] target/i386: Use DISAS_TOO_MANY to exit after gen_io_start We can set is_jmp early, using only one if, and let that be overwritten by gen_rep*'s calls to gen_jmp_tb. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-17-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 42 +++++++++---------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index be29ea7a03..11aaba8a65 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -5660,14 +5660,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_rdrand(s->T0, cpu_env); rm = (modrm & 7) | REX_B(s); gen_op_mov_reg_v(s, dflag, rm, s->T0); set_cc_op(s, CC_OP_EFLAGS); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; default: @@ -6704,15 +6702,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { gen_repz_ins(s, ot); - /* jump generated by gen_repz_ins */ } else { gen_ins(s, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } } break; case 0x6e: /* outsS */ @@ -6725,15 +6720,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { gen_repz_outs(s, ot); - /* jump generated by gen_repz_outs */ } else { gen_outs(s, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } } break; @@ -6750,13 +6742,11 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_in_func(ot, s->T1, s->tmp2_i32); gen_op_mov_reg_v(s, ot, R_EAX, s->T1); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0xe6: case 0xe7: @@ -6768,14 +6758,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_op_mov_v_reg(s, ot, s->T1, R_EAX); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_out_func(ot, s->tmp2_i32, s->tmp3_i32); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0xec: case 0xed: @@ -6787,13 +6775,11 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_in_func(ot, s->T1, s->tmp2_i32); gen_op_mov_reg_v(s, ot, R_EAX, s->T1); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0xee: case 0xef: @@ -6805,14 +6791,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_op_mov_v_reg(s, ot, s->T1, R_EAX); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_out_func(ot, s->tmp2_i32, s->tmp3_i32); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; /************************/ @@ -7478,11 +7462,9 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_update_eip_cur(s); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_rdtsc(cpu_env); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0x133: /* rdpmc */ gen_update_cc_op(s); @@ -7939,11 +7921,9 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_update_eip_cur(s); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_rdtscp(cpu_env); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; default: @@ -8307,6 +8287,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } if (b & 2) { gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0 + reg); @@ -8317,9 +8298,6 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_READ_CR0 + reg); gen_helper_read_crN(s->T0, cpu_env, tcg_constant_i32(reg)); gen_op_mov_reg_v(s, ot, rm, s->T0); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } } break; From 8760ded661c93b8aa76e74dcbd8c1a94764e8f12 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:26 -0700 Subject: [PATCH 22/37] target/i386: Create gen_jmp_rel Create a common helper for pc-relative branches. The jmp jb insn was missing a mask for CODE32. In all cases the CODE64 check was incorrectly placed, allowing PREFIX_DATA to truncate %rip to 16 bits. Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-18-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 58 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 11aaba8a65..ba1bd7c707 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -226,6 +226,7 @@ static void gen_eob(DisasContext *s); static void gen_jr(DisasContext *s); static void gen_jmp(DisasContext *s, target_ulong eip); static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num); +static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num); static void gen_op(DisasContext *s1, int op, MemOp ot, int d); static void gen_exception_gpf(DisasContext *s); @@ -2792,6 +2793,21 @@ static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num) } } +static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) +{ + target_ulong dest = s->pc - s->cs_base + diff; + + /* In 64-bit mode, operand size is fixed at 64 bits. */ + if (!CODE64(s)) { + if (ot == MO_16) { + dest &= 0xffff; + } else { + dest &= 0xffffffff; + } + } + gen_jmp_tb(s, dest, tb_num); +} + static void gen_jmp(DisasContext *s, target_ulong eip) { gen_jmp_tb(s, eip, 0); @@ -6862,20 +6878,12 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; case 0xe8: /* call im */ { - if (dflag != MO_16) { - tval = (int32_t)insn_get(env, s, MO_32); - } else { - tval = (int16_t)insn_get(env, s, MO_16); - } - tval += s->pc - s->cs_base; - if (dflag == MO_16) { - tval &= 0xffff; - } else if (!CODE64(s)) { - tval &= 0xffffffff; - } + int diff = (dflag != MO_16 + ? (int32_t)insn_get(env, s, MO_32) + : (int16_t)insn_get(env, s, MO_16)); gen_push_v(s, eip_next_tl(s)); gen_bnd_jmp(s); - gen_jmp(s, tval); + gen_jmp_rel(s, dflag, diff, 0); } break; case 0x9a: /* lcall im */ @@ -6893,19 +6901,13 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } goto do_lcall; case 0xe9: /* jmp im */ - if (dflag != MO_16) { - tval = (int32_t)insn_get(env, s, MO_32); - } else { - tval = (int16_t)insn_get(env, s, MO_16); + { + int diff = (dflag != MO_16 + ? (int32_t)insn_get(env, s, MO_32) + : (int16_t)insn_get(env, s, MO_16)); + gen_bnd_jmp(s); + gen_jmp_rel(s, dflag, diff, 0); } - tval += s->pc - s->cs_base; - if (dflag == MO_16) { - tval &= 0xffff; - } else if (!CODE64(s)) { - tval &= 0xffffffff; - } - gen_bnd_jmp(s); - gen_jmp(s, tval); break; case 0xea: /* ljmp im */ { @@ -6922,12 +6924,10 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } goto do_ljmp; case 0xeb: /* jmp Jb */ - tval = (int8_t)insn_get(env, s, MO_8); - tval += s->pc - s->cs_base; - if (dflag == MO_16) { - tval &= 0xffff; + { + int diff = (int8_t)insn_get(env, s, MO_8); + gen_jmp_rel(s, dflag, diff, 0); } - gen_jmp(s, tval); break; case 0x70 ... 0x7f: /* jcc Jb */ tval = (int8_t)insn_get(env, s, MO_8); From 2255da493a92e0f6441c773fa3bfdfb34273e85f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:27 -0700 Subject: [PATCH 23/37] target/i386: Use gen_jmp_rel for loop, repz, jecxz insns With gen_jmp_rel, we may chain to the next tb instead of merely writing to eip and exiting. For repz, subtract cur_insn_len to restart the current insn. Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-19-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index ba1bd7c707..434a6ad6cd 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -224,9 +224,9 @@ STUB_HELPER(wrmsr, TCGv_env env) static void gen_eob(DisasContext *s); static void gen_jr(DisasContext *s); -static void gen_jmp(DisasContext *s, target_ulong eip); static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num); static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num); +static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num); static void gen_op(DisasContext *s1, int op, MemOp ot, int d); static void gen_exception_gpf(DisasContext *s); @@ -1185,7 +1185,7 @@ static TCGLabel *gen_jz_ecx_string(DisasContext *s) TCGLabel *l2 = gen_new_label(); gen_op_jnz_ecx(s, s->aflag, l1); gen_set_label(l2); - gen_jmp_tb(s, s->pc - s->cs_base, 1); + gen_jmp_rel_csize(s, 0, 1); gen_set_label(l1); return l2; } @@ -1288,7 +1288,7 @@ static void gen_repz(DisasContext *s, MemOp ot, if (s->repz_opt) { gen_op_jz_ecx(s, s->aflag, l2); } - gen_jmp(s, s->base.pc_next - s->cs_base); + gen_jmp_rel_csize(s, -cur_insn_len(s), 0); } #define GEN_REPZ(op) \ @@ -1308,7 +1308,7 @@ static void gen_repz2(DisasContext *s, MemOp ot, int nz, if (s->repz_opt) { gen_op_jz_ecx(s, s->aflag, l2); } - gen_jmp(s, s->base.pc_next - s->cs_base); + gen_jmp_rel_csize(s, -cur_insn_len(s), 0); } #define GEN_REPZ2(op) \ @@ -2793,6 +2793,7 @@ static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num) } } +/* Jump to eip+diff, truncating the result to OT. */ static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) { target_ulong dest = s->pc - s->cs_base + diff; @@ -2808,9 +2809,11 @@ static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) gen_jmp_tb(s, dest, tb_num); } -static void gen_jmp(DisasContext *s, target_ulong eip) +/* Jump to eip+diff, truncating to the current code size. */ +static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num) { - gen_jmp_tb(s, eip, 0); + /* CODE64 ignores the OT argument, so we need not consider it. */ + gen_jmp_rel(s, CODE32(s) ? MO_32 : MO_16, diff, tb_num); } static inline void gen_ldq_env_A0(DisasContext *s, int offset) @@ -7404,24 +7407,18 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0xe2: /* loop */ case 0xe3: /* jecxz */ { - TCGLabel *l1, *l2, *l3; - - tval = (int8_t)insn_get(env, s, MO_8); - tval += s->pc - s->cs_base; - if (dflag == MO_16) { - tval &= 0xffff; - } + TCGLabel *l1, *l2; + int diff = (int8_t)insn_get(env, s, MO_8); l1 = gen_new_label(); l2 = gen_new_label(); - l3 = gen_new_label(); gen_update_cc_op(s); b &= 3; switch(b) { case 0: /* loopnz */ case 1: /* loopz */ gen_op_add_reg_im(s, s->aflag, R_ECX, -1); - gen_op_jz_ecx(s, s->aflag, l3); + gen_op_jz_ecx(s, s->aflag, l2); gen_jcc1(s, (JCC_Z << 1) | (b ^ 1), l1); break; case 2: /* loop */ @@ -7434,14 +7431,11 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) break; } - gen_set_label(l3); - gen_update_eip_next(s); - tcg_gen_br(l2); + gen_set_label(l2); + gen_jmp_rel_csize(s, 0, 1); gen_set_label(l1); - gen_jmp_im(s, tval); - gen_set_label(l2); - s->base.is_jmp = DISAS_EOB_ONLY; + gen_jmp_rel(s, dflag, diff, 0); } break; case 0x130: /* wrmsr */ From 54b191de675fbaee6886600c69786ad1c5fa158e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:28 -0700 Subject: [PATCH 24/37] target/i386: Use gen_jmp_rel for gen_jcc Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-20-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 57 ++++++++++++------------------------- 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 434a6ad6cd..5b84be4975 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -2409,32 +2409,14 @@ static void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) } } -static inline void gen_jcc(DisasContext *s, int b, - target_ulong val, target_ulong next_eip) +static void gen_jcc(DisasContext *s, int b, int diff) { - TCGLabel *l1, *l2; + TCGLabel *l1 = gen_new_label(); - if (s->jmp_opt) { - l1 = gen_new_label(); - gen_jcc1(s, b, l1); - - gen_goto_tb(s, 0, next_eip); - - gen_set_label(l1); - gen_goto_tb(s, 1, val); - } else { - l1 = gen_new_label(); - l2 = gen_new_label(); - gen_jcc1(s, b, l1); - - gen_jmp_im(s, next_eip); - tcg_gen_br(l2); - - gen_set_label(l1); - gen_jmp_im(s, val); - gen_set_label(l2); - gen_eob(s); - } + gen_jcc1(s, b, l1); + gen_jmp_rel_csize(s, 0, 1); + gen_set_label(l1); + gen_jmp_rel(s, s->dflag, diff, 0); } static void gen_cmovcc1(CPUX86State *env, DisasContext *s, MemOp ot, int b, @@ -4780,7 +4762,6 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) int shift; MemOp ot, aflag, dflag; int modrm, reg, rm, mod, op, opreg, val; - target_ulong next_eip, tval; bool orig_cc_op_dirty = s->cc_op_dirty; CCOp orig_cc_op = s->cc_op; @@ -6933,22 +6914,20 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) } break; case 0x70 ... 0x7f: /* jcc Jb */ - tval = (int8_t)insn_get(env, s, MO_8); - goto do_jcc; + { + int diff = (int8_t)insn_get(env, s, MO_8); + gen_bnd_jmp(s); + gen_jcc(s, b, diff); + } + break; case 0x180 ... 0x18f: /* jcc Jv */ - if (dflag != MO_16) { - tval = (int32_t)insn_get(env, s, MO_32); - } else { - tval = (int16_t)insn_get(env, s, MO_16); + { + int diff = (dflag != MO_16 + ? (int32_t)insn_get(env, s, MO_32) + : (int16_t)insn_get(env, s, MO_16)); + gen_bnd_jmp(s); + gen_jcc(s, b, diff); } - do_jcc: - next_eip = s->pc - s->cs_base; - tval += next_eip; - if (dflag == MO_16) { - tval &= 0xffff; - } - gen_bnd_jmp(s); - gen_jcc(s, b, tval, next_eip); break; case 0x190 ... 0x19f: /* setcc Gv */ From 5f7ec6efcc4fb5cc4cd8e8f2d61504d7909dc5b9 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:29 -0700 Subject: [PATCH 25/37] target/i386: Use gen_jmp_rel for DISAS_TOO_MANY With gen_jmp_rel, we may chain between two translation blocks which may only be separated because of TB size limits. Reviewed-by: Paolo Bonzini Signed-off-by: Richard Henderson Message-Id: <20221001140935.465607-21-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 5b84be4975..cf23ae6e5e 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -8798,6 +8798,9 @@ static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) case DISAS_NORETURN: break; case DISAS_TOO_MANY: + gen_update_cc_op(dc); + gen_jmp_rel_csize(dc, 0, 0); + break; case DISAS_EOB_NEXT: gen_update_cc_op(dc); gen_update_eip_cur(dc); From 0ebacb5d1e6c73365dd9d8435d4014fe0e326308 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:30 -0700 Subject: [PATCH 26/37] target/i386: Remove MemOp argument to gen_op_j*_ecx These functions are always passed aflag, so we might as well read it from DisasContext directly. While we're at it, use a common subroutine for these two functions. Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-22-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index cf23ae6e5e..9294f12f66 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -676,20 +676,21 @@ static void gen_exts(MemOp ot, TCGv reg) gen_ext_tl(reg, reg, ot, true); } -static inline -void gen_op_jnz_ecx(DisasContext *s, MemOp size, TCGLabel *label1) +static void gen_op_j_ecx(DisasContext *s, TCGCond cond, TCGLabel *label1) { tcg_gen_mov_tl(s->tmp0, cpu_regs[R_ECX]); - gen_extu(size, s->tmp0); - tcg_gen_brcondi_tl(TCG_COND_NE, s->tmp0, 0, label1); + gen_extu(s->aflag, s->tmp0); + tcg_gen_brcondi_tl(cond, s->tmp0, 0, label1); } -static inline -void gen_op_jz_ecx(DisasContext *s, MemOp size, TCGLabel *label1) +static inline void gen_op_jz_ecx(DisasContext *s, TCGLabel *label1) { - tcg_gen_mov_tl(s->tmp0, cpu_regs[R_ECX]); - gen_extu(size, s->tmp0); - tcg_gen_brcondi_tl(TCG_COND_EQ, s->tmp0, 0, label1); + gen_op_j_ecx(s, TCG_COND_EQ, label1); +} + +static inline void gen_op_jnz_ecx(DisasContext *s, TCGLabel *label1) +{ + gen_op_j_ecx(s, TCG_COND_NE, label1); } static void gen_helper_in_func(MemOp ot, TCGv v, TCGv_i32 n) @@ -1183,7 +1184,7 @@ static TCGLabel *gen_jz_ecx_string(DisasContext *s) { TCGLabel *l1 = gen_new_label(); TCGLabel *l2 = gen_new_label(); - gen_op_jnz_ecx(s, s->aflag, l1); + gen_op_jnz_ecx(s, l1); gen_set_label(l2); gen_jmp_rel_csize(s, 0, 1); gen_set_label(l1); @@ -1286,7 +1287,7 @@ static void gen_repz(DisasContext *s, MemOp ot, * before rep string_insn */ if (s->repz_opt) { - gen_op_jz_ecx(s, s->aflag, l2); + gen_op_jz_ecx(s, l2); } gen_jmp_rel_csize(s, -cur_insn_len(s), 0); } @@ -1306,7 +1307,7 @@ static void gen_repz2(DisasContext *s, MemOp ot, int nz, gen_update_cc_op(s); gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); if (s->repz_opt) { - gen_op_jz_ecx(s, s->aflag, l2); + gen_op_jz_ecx(s, l2); } gen_jmp_rel_csize(s, -cur_insn_len(s), 0); } @@ -7397,16 +7398,16 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) case 0: /* loopnz */ case 1: /* loopz */ gen_op_add_reg_im(s, s->aflag, R_ECX, -1); - gen_op_jz_ecx(s, s->aflag, l2); + gen_op_jz_ecx(s, l2); gen_jcc1(s, (JCC_Z << 1) | (b ^ 1), l1); break; case 2: /* loop */ gen_op_add_reg_im(s, s->aflag, R_ECX, -1); - gen_op_jnz_ecx(s, s->aflag, l1); + gen_op_jnz_ecx(s, l1); break; default: case 3: /* jcxz */ - gen_op_jz_ecx(s, s->aflag, l1); + gen_op_jz_ecx(s, l1); break; } From 900cc7e53679cbd70c8d5b718fc29873f77a0967 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:31 -0700 Subject: [PATCH 27/37] target/i386: Merge gen_jmp_tb and gen_goto_tb into gen_jmp_rel These functions have only one caller, and the logic is more obvious this way. Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-23-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 50 +++++++++++++------------------------ 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 9294f12f66..2e7b94700b 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -224,7 +224,6 @@ STUB_HELPER(wrmsr, TCGv_env env) static void gen_eob(DisasContext *s); static void gen_jr(DisasContext *s); -static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num); static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num); static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num); static void gen_op(DisasContext *s1, int op, MemOp ot, int d); @@ -2393,23 +2392,6 @@ static inline int insn_const_size(MemOp ot) } } -static void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) -{ - target_ulong pc = s->cs_base + eip; - - if (translator_use_goto_tb(&s->base, pc)) { - /* jump to same page: we can use a direct jump */ - tcg_gen_goto_tb(tb_num); - gen_jmp_im(s, eip); - tcg_gen_exit_tb(s->base.tb, tb_num); - s->base.is_jmp = DISAS_NORETURN; - } else { - /* jump to another page */ - gen_jmp_im(s, eip); - gen_jr(s); - } -} - static void gen_jcc(DisasContext *s, int b, int diff) { TCGLabel *l1 = gen_new_label(); @@ -2762,20 +2744,6 @@ static void gen_jr(DisasContext *s) do_gen_eob_worker(s, false, false, true); } -/* generate a jump to eip. No segment change must happen before as a - direct call to the next block may occur */ -static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num) -{ - gen_update_cc_op(s); - set_cc_op(s, CC_OP_DYNAMIC); - if (s->jmp_opt) { - gen_goto_tb(s, tb_num, eip); - } else { - gen_jmp_im(s, eip); - gen_eob(s); - } -} - /* Jump to eip+diff, truncating the result to OT. */ static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) { @@ -2789,7 +2757,23 @@ static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) dest &= 0xffffffff; } } - gen_jmp_tb(s, dest, tb_num); + + gen_update_cc_op(s); + set_cc_op(s, CC_OP_DYNAMIC); + if (!s->jmp_opt) { + gen_jmp_im(s, dest); + gen_eob(s); + } else if (translator_use_goto_tb(&s->base, dest)) { + /* jump to same page: we can use a direct jump */ + tcg_gen_goto_tb(tb_num); + gen_jmp_im(s, dest); + tcg_gen_exit_tb(s->base.tb, tb_num); + s->base.is_jmp = DISAS_NORETURN; + } else { + /* jump to another page */ + gen_jmp_im(s, dest); + gen_jr(s); + } } /* Jump to eip+diff, truncating to the current code size. */ From 75ec746a07b6db4c214102e644319a334c1ab899 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:32 -0700 Subject: [PATCH 28/37] target/i386: Create eip_cur_tl Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-24-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 2e7b94700b..5b0dab8633 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -562,6 +562,11 @@ static TCGv eip_next_tl(DisasContext *s) return tcg_constant_tl(s->pc - s->cs_base); } +static TCGv eip_cur_tl(DisasContext *s) +{ + return tcg_constant_tl(s->base.pc_next - s->cs_base); +} + /* Compute SEG:REG into A0. SEG is selected from the override segment (OVR_SEG) and the default segment (DEF_SEG). OVR_SEG may be -1 to indicate no override. */ @@ -6617,7 +6622,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) offsetof(CPUX86State, segs[R_CS].selector)); tcg_gen_st16_i32(s->tmp2_i32, cpu_env, offsetof(CPUX86State, fpcs)); - tcg_gen_st_tl(tcg_constant_tl(s->base.pc_next - s->cs_base), + tcg_gen_st_tl(eip_cur_tl(s), cpu_env, offsetof(CPUX86State, fpip)); } } From f771ca6a6127cdbdf4ed119e9d6c63abdad24181 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:33 -0700 Subject: [PATCH 29/37] target/i386: Add cpu_eip Create a tcg global temp for this, and use it instead of explicit stores. Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-25-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 5b0dab8633..f08fa060c4 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -64,6 +64,7 @@ /* global register indexes */ static TCGv cpu_cc_dst, cpu_cc_src, cpu_cc_src2; +static TCGv cpu_eip; static TCGv_i32 cpu_cc_op; static TCGv cpu_regs[CPU_NB_REGS]; static TCGv cpu_seg_base[6]; @@ -481,7 +482,7 @@ static void gen_add_A0_im(DisasContext *s, int val) static inline void gen_op_jmp_v(TCGv dest) { - tcg_gen_st_tl(dest, cpu_env, offsetof(CPUX86State, eip)); + tcg_gen_mov_tl(cpu_eip, dest); } static inline @@ -518,7 +519,7 @@ static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d) static void gen_jmp_im(DisasContext *s, target_ulong pc) { - gen_op_jmp_v(tcg_constant_tl(pc)); + tcg_gen_movi_tl(cpu_eip, pc); } static void gen_update_eip_cur(DisasContext *s) @@ -8614,6 +8615,13 @@ void tcg_x86_init(void) [R_EDI] = "edi", [R_EBP] = "ebp", [R_ESP] = "esp", +#endif + }; + static const char eip_name[] = { +#ifdef TARGET_X86_64 + "rip" +#else + "eip" #endif }; static const char seg_base_names[6][8] = { @@ -8640,6 +8648,7 @@ void tcg_x86_init(void) "cc_src"); cpu_cc_src2 = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_src2), "cc_src2"); + cpu_eip = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, eip), eip_name); for (i = 0; i < CPU_NB_REGS; ++i) { cpu_regs[i] = tcg_global_mem_new(cpu_env, From 7db973bece487cb0bb7976f5b67a8ae02fd32df2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:34 -0700 Subject: [PATCH 30/37] target/i386: Inline gen_jmp_im Expand this function at each of its callers. Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-26-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/tcg/translate.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index f08fa060c4..689a45256c 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -517,19 +517,14 @@ static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d) } } -static void gen_jmp_im(DisasContext *s, target_ulong pc) -{ - tcg_gen_movi_tl(cpu_eip, pc); -} - static void gen_update_eip_cur(DisasContext *s) { - gen_jmp_im(s, s->base.pc_next - s->cs_base); + tcg_gen_movi_tl(cpu_eip, s->base.pc_next - s->cs_base); } static void gen_update_eip_next(DisasContext *s) { - gen_jmp_im(s, s->pc - s->cs_base); + tcg_gen_movi_tl(cpu_eip, s->pc - s->cs_base); } static int cur_insn_len(DisasContext *s) @@ -2767,17 +2762,17 @@ static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) gen_update_cc_op(s); set_cc_op(s, CC_OP_DYNAMIC); if (!s->jmp_opt) { - gen_jmp_im(s, dest); + tcg_gen_movi_tl(cpu_eip, dest); gen_eob(s); } else if (translator_use_goto_tb(&s->base, dest)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); - gen_jmp_im(s, dest); + tcg_gen_movi_tl(cpu_eip, dest); tcg_gen_exit_tb(s->base.tb, tb_num); s->base.is_jmp = DISAS_NORETURN; } else { /* jump to another page */ - gen_jmp_im(s, dest); + tcg_gen_movi_tl(cpu_eip, dest); gen_jr(s); } } From e3a79e0e87831602e41819591a8e6dcc70a2a231 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 1 Oct 2022 07:09:35 -0700 Subject: [PATCH 31/37] target/i386: Enable TARGET_TB_PCREL Signed-off-by: Richard Henderson Reviewed-by: Paolo Bonzini Message-Id: <20221001140935.465607-27-richard.henderson@linaro.org> Signed-off-by: Paolo Bonzini --- target/i386/cpu-param.h | 4 ++ target/i386/tcg/tcg-cpu.c | 8 ++- target/i386/tcg/translate.c | 130 ++++++++++++++++++++++++++++-------- 3 files changed, 113 insertions(+), 29 deletions(-) diff --git a/target/i386/cpu-param.h b/target/i386/cpu-param.h index 9740bd7abd..1e79389761 100644 --- a/target/i386/cpu-param.h +++ b/target/i386/cpu-param.h @@ -25,4 +25,8 @@ #define TARGET_PAGE_BITS 12 #define NB_MMU_MODES 3 +#ifndef CONFIG_USER_ONLY +# define TARGET_TB_PCREL 1 +#endif + #endif diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index 6cf14c83ff..828244abe2 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -49,9 +49,11 @@ static void x86_cpu_exec_exit(CPUState *cs) static void x86_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { - X86CPU *cpu = X86_CPU(cs); - - cpu->env.eip = tb_pc(tb) - tb->cs_base; + /* The instruction pointer is always up to date with TARGET_TB_PCREL. */ + if (!TARGET_TB_PCREL) { + CPUX86State *env = cs->env_ptr; + env->eip = tb_pc(tb) - tb->cs_base; + } } #ifndef CONFIG_USER_ONLY diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 689a45256c..279a3ae999 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -78,6 +78,7 @@ typedef struct DisasContext { target_ulong pc; /* pc = eip + cs_base */ target_ulong cs_base; /* base of CS segment */ + target_ulong pc_save; MemOp aflag; MemOp dflag; @@ -480,9 +481,10 @@ static void gen_add_A0_im(DisasContext *s, int val) } } -static inline void gen_op_jmp_v(TCGv dest) +static inline void gen_op_jmp_v(DisasContext *s, TCGv dest) { tcg_gen_mov_tl(cpu_eip, dest); + s->pc_save = -1; } static inline @@ -519,12 +521,24 @@ static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d) static void gen_update_eip_cur(DisasContext *s) { - tcg_gen_movi_tl(cpu_eip, s->base.pc_next - s->cs_base); + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + tcg_gen_addi_tl(cpu_eip, cpu_eip, s->base.pc_next - s->pc_save); + } else { + tcg_gen_movi_tl(cpu_eip, s->base.pc_next - s->cs_base); + } + s->pc_save = s->base.pc_next; } static void gen_update_eip_next(DisasContext *s) { - tcg_gen_movi_tl(cpu_eip, s->pc - s->cs_base); + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + tcg_gen_addi_tl(cpu_eip, cpu_eip, s->pc - s->pc_save); + } else { + tcg_gen_movi_tl(cpu_eip, s->pc - s->cs_base); + } + s->pc_save = s->pc; } static int cur_insn_len(DisasContext *s) @@ -539,6 +553,7 @@ static TCGv_i32 cur_insn_len_i32(DisasContext *s) static TCGv_i32 eip_next_i32(DisasContext *s) { + assert(s->pc_save != -1); /* * This function has two users: lcall_real (always 16-bit mode), and * iret_protected (16, 32, or 64-bit mode). IRET only uses the value @@ -550,17 +565,38 @@ static TCGv_i32 eip_next_i32(DisasContext *s) if (CODE64(s)) { return tcg_constant_i32(-1); } - return tcg_constant_i32(s->pc - s->cs_base); + if (TARGET_TB_PCREL) { + TCGv_i32 ret = tcg_temp_new_i32(); + tcg_gen_trunc_tl_i32(ret, cpu_eip); + tcg_gen_addi_i32(ret, ret, s->pc - s->pc_save); + return ret; + } else { + return tcg_constant_i32(s->pc - s->cs_base); + } } static TCGv eip_next_tl(DisasContext *s) { - return tcg_constant_tl(s->pc - s->cs_base); + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + TCGv ret = tcg_temp_new(); + tcg_gen_addi_tl(ret, cpu_eip, s->pc - s->pc_save); + return ret; + } else { + return tcg_constant_tl(s->pc - s->cs_base); + } } static TCGv eip_cur_tl(DisasContext *s) { - return tcg_constant_tl(s->base.pc_next - s->cs_base); + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + TCGv ret = tcg_temp_new(); + tcg_gen_addi_tl(ret, cpu_eip, s->base.pc_next - s->pc_save); + return ret; + } else { + return tcg_constant_tl(s->base.pc_next - s->cs_base); + } } /* Compute SEG:REG into A0. SEG is selected from the override segment @@ -2260,7 +2296,12 @@ static TCGv gen_lea_modrm_1(DisasContext *s, AddressParts a) ea = cpu_regs[a.base]; } if (!ea) { - tcg_gen_movi_tl(s->A0, a.disp); + if (TARGET_TB_PCREL && a.base == -2) { + /* With cpu_eip ~= pc_save, the expression is pc-relative. */ + tcg_gen_addi_tl(s->A0, cpu_eip, a.disp - s->pc_save); + } else { + tcg_gen_movi_tl(s->A0, a.disp); + } ea = s->A0; } else if (a.disp != 0) { tcg_gen_addi_tl(s->A0, ea, a.disp); @@ -2748,32 +2789,58 @@ static void gen_jr(DisasContext *s) /* Jump to eip+diff, truncating the result to OT. */ static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) { - target_ulong dest = s->pc - s->cs_base + diff; + bool use_goto_tb = s->jmp_opt; + target_ulong mask = -1; + target_ulong new_pc = s->pc + diff; + target_ulong new_eip = new_pc - s->cs_base; /* In 64-bit mode, operand size is fixed at 64 bits. */ if (!CODE64(s)) { if (ot == MO_16) { - dest &= 0xffff; + mask = 0xffff; + if (TARGET_TB_PCREL && CODE32(s)) { + use_goto_tb = false; + } } else { - dest &= 0xffffffff; + mask = 0xffffffff; } } + new_eip &= mask; gen_update_cc_op(s); set_cc_op(s, CC_OP_DYNAMIC); - if (!s->jmp_opt) { - tcg_gen_movi_tl(cpu_eip, dest); - gen_eob(s); - } else if (translator_use_goto_tb(&s->base, dest)) { + + if (TARGET_TB_PCREL) { + tcg_gen_addi_tl(cpu_eip, cpu_eip, new_pc - s->pc_save); + /* + * If we can prove the branch does not leave the page and we have + * no extra masking to apply (data16 branch in code32, see above), + * then we have also proven that the addition does not wrap. + */ + if (!use_goto_tb || !is_same_page(&s->base, new_pc)) { + tcg_gen_andi_tl(cpu_eip, cpu_eip, mask); + use_goto_tb = false; + } + } + + if (use_goto_tb && + translator_use_goto_tb(&s->base, new_eip + s->cs_base)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); - tcg_gen_movi_tl(cpu_eip, dest); + if (!TARGET_TB_PCREL) { + tcg_gen_movi_tl(cpu_eip, new_eip); + } tcg_gen_exit_tb(s->base.tb, tb_num); s->base.is_jmp = DISAS_NORETURN; } else { - /* jump to another page */ - tcg_gen_movi_tl(cpu_eip, dest); - gen_jr(s); + if (!TARGET_TB_PCREL) { + tcg_gen_movi_tl(cpu_eip, new_eip); + } + if (s->jmp_opt) { + gen_jr(s); /* jump to another page */ + } else { + gen_eob(s); /* exit to main loop */ + } } } @@ -5329,7 +5396,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_ext16u_tl(s->T0, s->T0); } gen_push_v(s, eip_next_tl(s)); - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); s->base.is_jmp = DISAS_JUMP; break; @@ -5359,7 +5426,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) if (dflag == MO_16) { tcg_gen_ext16u_tl(s->T0, s->T0); } - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); s->base.is_jmp = DISAS_JUMP; break; @@ -5377,7 +5444,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) eip_next_tl(s)); } else { gen_op_movl_seg_T0_vm(s, R_CS); - gen_op_jmp_v(s->T1); + gen_op_jmp_v(s, s->T1); } s->base.is_jmp = DISAS_JUMP; break; @@ -6792,7 +6859,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) ot = gen_pop_T0(s); gen_stack_update(s, val + (1 << ot)); /* Note that gen_pop_T0 uses a zero-extending load. */ - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); s->base.is_jmp = DISAS_JUMP; break; @@ -6800,7 +6867,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) ot = gen_pop_T0(s); gen_pop_update(s, ot); /* Note that gen_pop_T0 uses a zero-extending load. */ - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); s->base.is_jmp = DISAS_JUMP; break; @@ -6818,7 +6885,7 @@ static bool disas_insn(DisasContext *s, CPUState *cpu) gen_op_ld_v(s, dflag, s->T0, s->A0); /* NOTE: keeping EIP updated is not a problem in case of exception */ - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); /* pop selector */ gen_add_A0_im(s, 1 << dflag); gen_op_ld_v(s, dflag, s->T0, s->A0); @@ -8680,6 +8747,7 @@ static void i386_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cpu) int iopl = (flags >> IOPL_SHIFT) & 3; dc->cs_base = dc->base.tb->cs_base; + dc->pc_save = dc->base.pc_next; dc->flags = flags; #ifndef CONFIG_USER_ONLY dc->cpl = cpl; @@ -8743,9 +8811,14 @@ static void i386_tr_tb_start(DisasContextBase *db, CPUState *cpu) static void i386_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); + target_ulong pc_arg = dc->base.pc_next; dc->prev_insn_end = tcg_last_op(); - tcg_gen_insn_start(dc->base.pc_next, dc->cc_op); + if (TARGET_TB_PCREL) { + pc_arg -= dc->cs_base; + pc_arg &= ~TARGET_PAGE_MASK; + } + tcg_gen_insn_start(pc_arg, dc->cc_op); } static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) @@ -8846,7 +8919,12 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, target_ulong *data) { int cc_op = data[1]; - env->eip = data[0] - tb->cs_base; + + if (TARGET_TB_PCREL) { + env->eip = (env->eip & TARGET_PAGE_MASK) | data[0]; + } else { + env->eip = data[0] - tb->cs_base; + } if (cc_op != CC_OP_DYNAMIC) { env->cc_op = cc_op; } From 62a44fddb24fec35a6baf7e2c52b0e935a5bfa90 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 5 Oct 2022 00:56:41 +0200 Subject: [PATCH 32/37] x86: Implement MSR_CORE_THREAD_COUNT MSR Intel CPUs starting with Haswell-E implement a new MSR called MSR_CORE_THREAD_COUNT which exposes the number of threads and cores inside of a package. This MSR is used by XNU to populate internal data structures and not implementing it prevents virtual machines with more than 1 vCPU from booting if the emulated CPU generation is at least Haswell-E. This patch propagates the existing hvf logic from patch 027ac0cb516 ("target/i386/hvf: add rdmsr 35H MSR_CORE_THREAD_COUNT") to TCG. Signed-off-by: Alexander Graf Message-Id: <20221004225643.65036-2-agraf@csgraf.de> Signed-off-by: Paolo Bonzini --- target/i386/tcg/sysemu/misc_helper.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/target/i386/tcg/sysemu/misc_helper.c b/target/i386/tcg/sysemu/misc_helper.c index 1328aa656f..e1528b7f80 100644 --- a/target/i386/tcg/sysemu/misc_helper.c +++ b/target/i386/tcg/sysemu/misc_helper.c @@ -450,6 +450,11 @@ void helper_rdmsr(CPUX86State *env) case MSR_IA32_UCODE_REV: val = x86_cpu->ucode_rev; break; + case MSR_CORE_THREAD_COUNT: { + CPUState *cs = CPU(x86_cpu); + val = (cs->nr_threads * cs->nr_cores) | (cs->nr_cores << 16); + break; + } default: if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL + From 860054d8ce4067ef2bc3deb2a98cf93350fc03e4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 5 Oct 2022 00:56:42 +0200 Subject: [PATCH 33/37] i386: kvm: Add support for MSR filtering KVM has grown support to deflect arbitrary MSRs to user space since Linux 5.10. For now we don't expect to make a lot of use of this feature, so let's expose it the easiest way possible: With up to 16 individually maskable MSRs. This patch adds a kvm_filter_msr() function that other code can call to install a hook on KVM MSR reads or writes. Signed-off-by: Alexander Graf Message-Id: <20221004225643.65036-3-agraf@csgraf.de> Signed-off-by: Paolo Bonzini --- target/i386/kvm/kvm.c | 123 +++++++++++++++++++++++++++++++++++++ target/i386/kvm/kvm_i386.h | 11 ++++ 2 files changed, 134 insertions(+) diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index ec63b5eb10..1d9a50b02b 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -141,6 +141,8 @@ static struct kvm_cpuid2 *cpuid_cache; static struct kvm_cpuid2 *hv_cpuid_cache; static struct kvm_msr_list *kvm_feature_msrs; +static KVMMSRHandlers msr_handlers[KVM_MSR_FILTER_MAX_RANGES]; + #define BUS_LOCK_SLICE_TIME 1000000000ULL /* ns */ static RateLimit bus_lock_ratelimit_ctrl; static int kvm_get_one_msr(X86CPU *cpu, int index, uint64_t *value); @@ -2610,6 +2612,15 @@ int kvm_arch_init(MachineState *ms, KVMState *s) return ret; } } + if (kvm_vm_check_extension(s, KVM_CAP_X86_USER_SPACE_MSR)) { + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_USER_SPACE_MSR, 0, + KVM_MSR_EXIT_REASON_FILTER); + if (ret) { + error_report("Could not enable user space MSRs: %s", + strerror(-ret)); + exit(1); + } + } return 0; } @@ -5109,6 +5120,108 @@ void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg) } } +static bool kvm_install_msr_filters(KVMState *s) +{ + uint64_t zero = 0; + struct kvm_msr_filter filter = { + .flags = KVM_MSR_FILTER_DEFAULT_ALLOW, + }; + int r, i, j = 0; + + for (i = 0; i < KVM_MSR_FILTER_MAX_RANGES; i++) { + KVMMSRHandlers *handler = &msr_handlers[i]; + if (handler->msr) { + struct kvm_msr_filter_range *range = &filter.ranges[j++]; + + *range = (struct kvm_msr_filter_range) { + .flags = 0, + .nmsrs = 1, + .base = handler->msr, + .bitmap = (__u8 *)&zero, + }; + + if (handler->rdmsr) { + range->flags |= KVM_MSR_FILTER_READ; + } + + if (handler->wrmsr) { + range->flags |= KVM_MSR_FILTER_WRITE; + } + } + } + + r = kvm_vm_ioctl(s, KVM_X86_SET_MSR_FILTER, &filter); + if (r) { + return false; + } + + return true; +} + +bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr, + QEMUWRMSRHandler *wrmsr) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) { + if (!msr_handlers[i].msr) { + msr_handlers[i] = (KVMMSRHandlers) { + .msr = msr, + .rdmsr = rdmsr, + .wrmsr = wrmsr, + }; + + if (!kvm_install_msr_filters(s)) { + msr_handlers[i] = (KVMMSRHandlers) { }; + return false; + } + + return true; + } + } + + return false; +} + +static int kvm_handle_rdmsr(X86CPU *cpu, struct kvm_run *run) +{ + int i; + bool r; + + for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) { + KVMMSRHandlers *handler = &msr_handlers[i]; + if (run->msr.index == handler->msr) { + if (handler->rdmsr) { + r = handler->rdmsr(cpu, handler->msr, + (uint64_t *)&run->msr.data); + run->msr.error = r ? 0 : 1; + return 0; + } + } + } + + assert(false); +} + +static int kvm_handle_wrmsr(X86CPU *cpu, struct kvm_run *run) +{ + int i; + bool r; + + for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) { + KVMMSRHandlers *handler = &msr_handlers[i]; + if (run->msr.index == handler->msr) { + if (handler->wrmsr) { + r = handler->wrmsr(cpu, handler->msr, run->msr.data); + run->msr.error = r ? 0 : 1; + return 0; + } + } + } + + assert(false); +} + static bool has_sgx_provisioning; static bool __kvm_enable_sgx_provisioning(KVMState *s) @@ -5226,6 +5339,16 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) ret = 0; } break; + case KVM_EXIT_X86_RDMSR: + /* We only enable MSR filtering, any other exit is bogus */ + assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER); + ret = kvm_handle_rdmsr(cpu, run); + break; + case KVM_EXIT_X86_WRMSR: + /* We only enable MSR filtering, any other exit is bogus */ + assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER); + ret = kvm_handle_wrmsr(cpu, run); + break; default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); ret = -1; diff --git a/target/i386/kvm/kvm_i386.h b/target/i386/kvm/kvm_i386.h index 4124912c20..2ed586c11b 100644 --- a/target/i386/kvm/kvm_i386.h +++ b/target/i386/kvm/kvm_i386.h @@ -54,4 +54,15 @@ uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address); bool kvm_enable_sgx_provisioning(KVMState *s); void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask); +typedef bool QEMURDMSRHandler(X86CPU *cpu, uint32_t msr, uint64_t *val); +typedef bool QEMUWRMSRHandler(X86CPU *cpu, uint32_t msr, uint64_t val); +typedef struct kvm_msr_handlers { + uint32_t msr; + QEMURDMSRHandler *rdmsr; + QEMUWRMSRHandler *wrmsr; +} KVMMSRHandlers; + +bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr, + QEMUWRMSRHandler *wrmsr); + #endif From 37656470f67398dd10101d7d940d660053f60ff5 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 5 Oct 2022 00:56:43 +0200 Subject: [PATCH 34/37] KVM: x86: Implement MSR_CORE_THREAD_COUNT MSR The MSR_CORE_THREAD_COUNT MSR describes CPU package topology, such as number of threads and cores for a given package. This is information that QEMU has readily available and can provide through the new user space MSR deflection interface. This patch propagates the existing hvf logic from patch 027ac0cb516 ("target/i386/hvf: add rdmsr 35H MSR_CORE_THREAD_COUNT") to KVM. Signed-off-by: Alexander Graf Message-Id: <20221004225643.65036-4-agraf@csgraf.de> Signed-off-by: Paolo Bonzini --- target/i386/kvm/kvm.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index 1d9a50b02b..bed6c00f2c 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -2401,6 +2401,17 @@ static int kvm_get_supported_msrs(KVMState *s) return ret; } +static bool kvm_rdmsr_core_thread_count(X86CPU *cpu, uint32_t msr, + uint64_t *val) +{ + CPUState *cs = CPU(cpu); + + *val = cs->nr_threads * cs->nr_cores; /* thread count, bits 15..0 */ + *val |= ((uint32_t)cs->nr_cores << 16); /* core count, bits 31..16 */ + + return true; +} + static Notifier smram_machine_done; static KVMMemoryListener smram_listener; static AddressSpace smram_address_space; @@ -2613,6 +2624,8 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } if (kvm_vm_check_extension(s, KVM_CAP_X86_USER_SPACE_MSR)) { + bool r; + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_USER_SPACE_MSR, 0, KVM_MSR_EXIT_REASON_FILTER); if (ret) { @@ -2620,6 +2633,14 @@ int kvm_arch_init(MachineState *ms, KVMState *s) strerror(-ret)); exit(1); } + + r = kvm_filter_msr(s, MSR_CORE_THREAD_COUNT, + kvm_rdmsr_core_thread_count, NULL); + if (!r) { + error_report("Could not install MSR_CORE_THREAD_COUNT handler: %s", + strerror(-ret)); + exit(1); + } } return 0; From 5154d35beda383af8042061099a5cd2decf88e69 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 7 Oct 2022 15:47:35 +0200 Subject: [PATCH 35/37] linux-user: i386/signal: move fpstate at the end of the 32-bit frames Recent versions of Linux moved the 32-bit fpstate towards the end of the frame, so that the variable-sized xsave data does not overwrite the (ABI-defined) extramask[] field. Follow suit in QEMU. Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- linux-user/i386/signal.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/linux-user/i386/signal.c b/linux-user/i386/signal.c index 4372621a4d..76317a3d16 100644 --- a/linux-user/i386/signal.c +++ b/linux-user/i386/signal.c @@ -163,9 +163,16 @@ struct sigframe { abi_ulong pretcode; int sig; struct target_sigcontext sc; - struct target_fpstate fpstate; + /* + * The actual fpstate is placed after retcode[] below, to make + * room for the variable-sized xsave data. The older unused fpstate + * has to be kept to avoid changing the offset of extramask[], which + * is part of the ABI. + */ + struct target_fpstate fpstate_unused; abi_ulong extramask[TARGET_NSIG_WORDS-1]; char retcode[8]; + struct target_fpstate fpstate; }; struct rt_sigframe { @@ -175,8 +182,8 @@ struct rt_sigframe { abi_ulong puc; struct target_siginfo info; struct target_ucontext uc; - struct target_fpstate fpstate; char retcode[8]; + struct target_fpstate fpstate; }; #else From 2796f290b5469a7f6749ea119a48bc17f489effd Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 6 Oct 2022 13:31:06 +0200 Subject: [PATCH 36/37] linux-user: i386/signal: support FXSAVE fpstate on 32-bit emulation Linux can use FXSAVE to save/restore XMM registers even on 32-bit systems. This requires some care in order to keep the FXSAVE area aligned to 16 bytes; for this reason, get_sigframe is changed to pass the offset into the FXSAVE area rather than the full frame size. Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- linux-user/i386/signal.c | 129 +++++++++++++++++++++++---------------- 1 file changed, 77 insertions(+), 52 deletions(-) diff --git a/linux-user/i386/signal.c b/linux-user/i386/signal.c index 76317a3d16..58b11be116 100644 --- a/linux-user/i386/signal.c +++ b/linux-user/i386/signal.c @@ -39,29 +39,7 @@ struct target_xmmreg { uint32_t element[4]; }; -struct target_fpstate_32 { - /* Regular FPU environment */ - uint32_t cw; - uint32_t sw; - uint32_t tag; - uint32_t ipoff; - uint32_t cssel; - uint32_t dataoff; - uint32_t datasel; - struct target_fpreg st[8]; - uint16_t status; - uint16_t magic; /* 0xffff = regular FPU data only */ - - /* FXSR FPU environment */ - uint32_t _fxsr_env[6]; /* FXSR FPU env is ignored */ - uint32_t mxcsr; - uint32_t reserved; - struct target_fpxreg fxsr_st[8]; /* FXSR FPU reg data is ignored */ - struct target_xmmreg xmm[8]; - uint32_t padding[56]; -}; - -struct target_fpstate_64 { +struct target_fpstate_fxsave { /* FXSAVE format */ uint16_t cw; uint16_t sw; @@ -75,11 +53,36 @@ struct target_fpstate_64 { uint32_t xmm_space[64]; uint32_t reserved[24]; }; +#define TARGET_FXSAVE_SIZE sizeof(struct target_fpstate_fxsave) +QEMU_BUILD_BUG_ON(TARGET_FXSAVE_SIZE != 512); + +struct target_fpstate_32 { + /* Regular FPU environment */ + uint32_t cw; + uint32_t sw; + uint32_t tag; + uint32_t ipoff; + uint32_t cssel; + uint32_t dataoff; + uint32_t datasel; + struct target_fpreg st[8]; + uint16_t status; + uint16_t magic; /* 0xffff = regular FPU data only */ + struct target_fpstate_fxsave fxsave; +}; + +/* + * For simplicity, setup_frame aligns struct target_fpstate_32 to + * 16 bytes, so ensure that the FXSAVE area is also aligned. + */ +QEMU_BUILD_BUG_ON(offsetof(struct target_fpstate_32, fxsave) & 15); #ifndef TARGET_X86_64 # define target_fpstate target_fpstate_32 +# define TARGET_FPSTATE_FXSAVE_OFFSET offsetof(struct target_fpstate_32, fxsave) #else -# define target_fpstate target_fpstate_64 +# define target_fpstate target_fpstate_fxsave +# define TARGET_FPSTATE_FXSAVE_OFFSET 0 #endif struct target_sigcontext_32 { @@ -172,8 +175,16 @@ struct sigframe { struct target_fpstate fpstate_unused; abi_ulong extramask[TARGET_NSIG_WORDS-1]; char retcode[8]; - struct target_fpstate fpstate; + + /* + * This field will be 16-byte aligned in memory. Applying QEMU_ALIGNED + * to it ensures that the base of the frame has an appropriate alignment + * too. + */ + struct target_fpstate fpstate QEMU_ALIGNED(8); }; +#define TARGET_SIGFRAME_FXSAVE_OFFSET ( \ + offsetof(struct sigframe, fpstate) + TARGET_FPSTATE_FXSAVE_OFFSET) struct rt_sigframe { abi_ulong pretcode; @@ -183,25 +194,35 @@ struct rt_sigframe { struct target_siginfo info; struct target_ucontext uc; char retcode[8]; - struct target_fpstate fpstate; + struct target_fpstate fpstate QEMU_ALIGNED(8); }; - +#define TARGET_RT_SIGFRAME_FXSAVE_OFFSET ( \ + offsetof(struct rt_sigframe, fpstate) + TARGET_FPSTATE_FXSAVE_OFFSET) #else struct rt_sigframe { abi_ulong pretcode; struct target_ucontext uc; struct target_siginfo info; - struct target_fpstate fpstate; + struct target_fpstate fpstate QEMU_ALIGNED(16); }; - +#define TARGET_RT_SIGFRAME_FXSAVE_OFFSET ( \ + offsetof(struct rt_sigframe, fpstate) + TARGET_FPSTATE_FXSAVE_OFFSET) #endif /* * Set up a signal frame. */ -/* XXX: save x87 state */ +static void fxsave_sigcontext(CPUX86State *env, struct target_fpstate_fxsave *fxsave, + abi_ulong fxsave_addr) +{ + /* fxsave_addr must be 16 byte aligned for fxsave */ + assert(!(fxsave_addr & 0xf)); + + cpu_x86_fxsave(env, fxsave_addr); +} + static void setup_sigcontext(struct target_sigcontext *sc, struct target_fpstate *fpstate, CPUX86State *env, abi_ulong mask, abi_ulong fpstate_addr) @@ -233,13 +254,14 @@ static void setup_sigcontext(struct target_sigcontext *sc, cpu_x86_fsave(env, fpstate_addr, 1); fpstate->status = fpstate->sw; - magic = 0xffff; + if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { + magic = 0xffff; + } else { + fxsave_sigcontext(env, &fpstate->fxsave, + fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); + magic = 0; + } __put_user(magic, &fpstate->magic); - __put_user(fpstate_addr, &sc->fpstate); - - /* non-iBCS2 extensions.. */ - __put_user(mask, &sc->oldmask); - __put_user(env->cr[2], &sc->cr2); #else __put_user(env->regs[R_EDI], &sc->rdi); __put_user(env->regs[R_ESI], &sc->rsi); @@ -269,15 +291,14 @@ static void setup_sigcontext(struct target_sigcontext *sc, __put_user((uint16_t)0, &sc->fs); __put_user(env->segs[R_SS].selector, &sc->ss); + fxsave_sigcontext(env, fpstate, fpstate_addr); +#endif + + __put_user(fpstate_addr, &sc->fpstate); + + /* non-iBCS2 extensions.. */ __put_user(mask, &sc->oldmask); __put_user(env->cr[2], &sc->cr2); - - /* fpstate_addr must be 16 byte aligned for fxsave */ - assert(!(fpstate_addr & 0xf)); - - cpu_x86_fxsave(env, fpstate_addr); - __put_user(fpstate_addr, &sc->fpstate); -#endif } /* @@ -285,7 +306,7 @@ static void setup_sigcontext(struct target_sigcontext *sc, */ static inline abi_ulong -get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t frame_size) +get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t fxsave_offset) { unsigned long esp; @@ -309,11 +330,11 @@ get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t frame_size) #endif } -#ifndef TARGET_X86_64 - return (esp - frame_size) & -8ul; -#else - return ((esp - frame_size) & (~15ul)) - 8; -#endif + if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { + return (esp - (fxsave_offset + TARGET_FXSAVE_SIZE)) & -8ul; + } else { + return ((esp - TARGET_FXSAVE_SIZE) & -16ul) - fxsave_offset; + } } #ifndef TARGET_X86_64 @@ -341,7 +362,7 @@ void setup_frame(int sig, struct target_sigaction *ka, struct sigframe *frame; int i; - frame_addr = get_sigframe(ka, env, sizeof(*frame)); + frame_addr = get_sigframe(ka, env, TARGET_SIGFRAME_FXSAVE_OFFSET); trace_user_setup_frame(env, frame_addr); if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) @@ -397,7 +418,7 @@ void setup_rt_frame(int sig, struct target_sigaction *ka, struct rt_sigframe *frame; int i; - frame_addr = get_sigframe(ka, env, sizeof(*frame)); + frame_addr = get_sigframe(ka, env, TARGET_RT_SIGFRAME_FXSAVE_OFFSET); trace_user_setup_rt_frame(env, frame_addr); if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) @@ -529,7 +550,11 @@ restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc) goto badframe; } #ifndef TARGET_X86_64 - cpu_x86_frstor(env, fpstate_addr, 1); + if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { + cpu_x86_frstor(env, fpstate_addr, 1); + } else { + cpu_x86_fxrstor(env, fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); + } #else cpu_x86_fxrstor(env, fpstate_addr); #endif From 5d2456789ac50b11c2bd560ddf3470fe820bb0ff Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 10 Oct 2022 15:28:54 +0200 Subject: [PATCH 37/37] linux-user: i386/signal: support XSAVE/XRSTOR for signal frame fpstate Add support for saving/restoring extended save states when signals are delivered. This allows using AVX, MPX or PKRU registers in signal handlers. Reviewed-by: Richard Henderson Signed-off-by: Paolo Bonzini --- linux-user/i386/signal.c | 119 +++++++++++++++++++++++++++++------ target/i386/cpu.c | 2 +- target/i386/cpu.h | 3 + target/i386/tcg/fpu_helper.c | 64 +++++++++++-------- 4 files changed, 142 insertions(+), 46 deletions(-) diff --git a/linux-user/i386/signal.c b/linux-user/i386/signal.c index 58b11be116..60fa07d6f9 100644 --- a/linux-user/i386/signal.c +++ b/linux-user/i386/signal.c @@ -24,6 +24,10 @@ /* from the Linux kernel - /arch/x86/include/uapi/asm/sigcontext.h */ +#define TARGET_FP_XSTATE_MAGIC1 0x46505853U /* FPXS */ +#define TARGET_FP_XSTATE_MAGIC2 0x46505845U /* FPXE */ +#define TARGET_FP_XSTATE_MAGIC2_SIZE 4 + struct target_fpreg { uint16_t significand[4]; uint16_t exponent; @@ -39,6 +43,15 @@ struct target_xmmreg { uint32_t element[4]; }; +struct target_fpx_sw_bytes { + uint32_t magic1; + uint32_t extended_size; + uint64_t xfeatures; + uint32_t xstate_size; + uint32_t reserved[7]; +}; +QEMU_BUILD_BUG_ON(sizeof(struct target_fpx_sw_bytes) != 12*4); + struct target_fpstate_fxsave { /* FXSAVE format */ uint16_t cw; @@ -51,10 +64,13 @@ struct target_fpstate_fxsave { uint32_t mxcsr_mask; uint32_t st_space[32]; uint32_t xmm_space[64]; - uint32_t reserved[24]; + uint32_t hw_reserved[12]; + struct target_fpx_sw_bytes sw_reserved; + uint8_t xfeatures[]; }; #define TARGET_FXSAVE_SIZE sizeof(struct target_fpstate_fxsave) QEMU_BUILD_BUG_ON(TARGET_FXSAVE_SIZE != 512); +QEMU_BUILD_BUG_ON(offsetof(struct target_fpstate_fxsave, sw_reserved) != 464); struct target_fpstate_32 { /* Regular FPU environment */ @@ -214,13 +230,39 @@ struct rt_sigframe { * Set up a signal frame. */ -static void fxsave_sigcontext(CPUX86State *env, struct target_fpstate_fxsave *fxsave, - abi_ulong fxsave_addr) +static void xsave_sigcontext(CPUX86State *env, struct target_fpstate_fxsave *fxsave, + abi_ulong fxsave_addr) { - /* fxsave_addr must be 16 byte aligned for fxsave */ - assert(!(fxsave_addr & 0xf)); + if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) { + /* fxsave_addr must be 16 byte aligned for fxsave */ + assert(!(fxsave_addr & 0xf)); - cpu_x86_fxsave(env, fxsave_addr); + cpu_x86_fxsave(env, fxsave_addr); + __put_user(0, &fxsave->sw_reserved.magic1); + } else { + uint32_t xstate_size = xsave_area_size(env->xcr0, false); + uint32_t xfeatures_size = xstate_size - TARGET_FXSAVE_SIZE; + + /* + * extended_size is the offset from fpstate_addr to right after the end + * of the extended save states. On 32-bit that includes the legacy + * FSAVE area. + */ + uint32_t extended_size = TARGET_FPSTATE_FXSAVE_OFFSET + + xstate_size + TARGET_FP_XSTATE_MAGIC2_SIZE; + + /* fxsave_addr must be 64 byte aligned for xsave */ + assert(!(fxsave_addr & 0x3f)); + + /* Zero the header, XSAVE *adds* features to an existing save state. */ + memset(fxsave->xfeatures, 0, 64); + cpu_x86_xsave(env, fxsave_addr); + __put_user(TARGET_FP_XSTATE_MAGIC1, &fxsave->sw_reserved.magic1); + __put_user(extended_size, &fxsave->sw_reserved.extended_size); + __put_user(env->xcr0, &fxsave->sw_reserved.xfeatures); + __put_user(xstate_size, &fxsave->sw_reserved.xstate_size); + __put_user(TARGET_FP_XSTATE_MAGIC2, (uint32_t *) &fxsave->xfeatures[xfeatures_size]); + } } static void setup_sigcontext(struct target_sigcontext *sc, @@ -257,8 +299,8 @@ static void setup_sigcontext(struct target_sigcontext *sc, if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { magic = 0xffff; } else { - fxsave_sigcontext(env, &fpstate->fxsave, - fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); + xsave_sigcontext(env, &fpstate->fxsave, + fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); magic = 0; } __put_user(magic, &fpstate->magic); @@ -291,7 +333,7 @@ static void setup_sigcontext(struct target_sigcontext *sc, __put_user((uint16_t)0, &sc->fs); __put_user(env->segs[R_SS].selector, &sc->ss); - fxsave_sigcontext(env, fpstate, fpstate_addr); + xsave_sigcontext(env, fpstate, fpstate_addr); #endif __put_user(fpstate_addr, &sc->fpstate); @@ -332,8 +374,12 @@ get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t fxsave_offset if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { return (esp - (fxsave_offset + TARGET_FXSAVE_SIZE)) & -8ul; - } else { + } else if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) { return ((esp - TARGET_FXSAVE_SIZE) & -16ul) - fxsave_offset; + } else { + size_t xstate_size = + xsave_area_size(env->xcr0, false) + TARGET_FP_XSTATE_MAGIC2_SIZE; + return ((esp - xstate_size) & -64ul) - fxsave_offset; } } @@ -437,7 +483,11 @@ void setup_rt_frame(int sig, struct target_sigaction *ka, } /* Create the ucontext. */ - __put_user(0, &frame->uc.tuc_flags); + if (env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) { + __put_user(1, &frame->uc.tuc_flags); + } else { + __put_user(0, &frame->uc.tuc_flags); + } __put_user(0, &frame->uc.tuc_link); target_save_altstack(&frame->uc.tuc_stack, env); setup_sigcontext(&frame->uc.tuc_mcontext, &frame->fpstate, env, @@ -491,10 +541,37 @@ give_sigsegv: force_sigsegv(sig); } +static int xrstor_sigcontext(CPUX86State *env, struct target_fpstate_fxsave *fxsave, + abi_ulong fxsave_addr) +{ + if (env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) { + uint32_t extended_size = tswapl(fxsave->sw_reserved.extended_size); + uint32_t xstate_size = tswapl(fxsave->sw_reserved.xstate_size); + uint32_t xfeatures_size = xstate_size - TARGET_FXSAVE_SIZE; + + /* Linux checks MAGIC2 using xstate_size, not extended_size. */ + if (tswapl(fxsave->sw_reserved.magic1) == TARGET_FP_XSTATE_MAGIC1 && + extended_size >= TARGET_FPSTATE_FXSAVE_OFFSET + xstate_size + TARGET_FP_XSTATE_MAGIC2_SIZE) { + if (!access_ok(env_cpu(env), VERIFY_READ, fxsave_addr, + extended_size - TARGET_FPSTATE_FXSAVE_OFFSET)) { + return 1; + } + if (tswapl(*(uint32_t *) &fxsave->xfeatures[xfeatures_size]) == TARGET_FP_XSTATE_MAGIC2) { + cpu_x86_xrstor(env, fxsave_addr); + return 0; + } + } + /* fall through to fxrstor */ + } + + cpu_x86_fxrstor(env, fxsave_addr); + return 0; +} + static int restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc) { - unsigned int err = 0; + int err = 1; abi_ulong fpstate_addr; unsigned int tmpflags; @@ -545,24 +622,28 @@ restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc) fpstate_addr = tswapl(sc->fpstate); if (fpstate_addr != 0) { - if (!access_ok(env_cpu(env), VERIFY_READ, fpstate_addr, - sizeof(struct target_fpstate))) { - goto badframe; + struct target_fpstate *fpstate; + if (!lock_user_struct(VERIFY_READ, fpstate, fpstate_addr, + sizeof(struct target_fpstate))) { + return err; } #ifndef TARGET_X86_64 if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { cpu_x86_frstor(env, fpstate_addr, 1); + err = 0; } else { - cpu_x86_fxrstor(env, fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); + err = xrstor_sigcontext(env, &fpstate->fxsave, + fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); } #else - cpu_x86_fxrstor(env, fpstate_addr); + err = xrstor_sigcontext(env, fpstate, fpstate_addr); #endif + unlock_user_struct(fpstate, fpstate_addr, 0); + } else { + err = 0; } return err; -badframe: - return 1; } /* Note: there is no sigreturn on x86_64, there is only rt_sigreturn */ diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 06884177fa..8a11470507 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -1467,7 +1467,7 @@ ExtSaveArea x86_ext_save_areas[XSAVE_STATE_AREA_COUNT] = { }, }; -static uint32_t xsave_area_size(uint64_t mask, bool compacted) +uint32_t xsave_area_size(uint64_t mask, bool compacted) { uint64_t ret = x86_ext_save_areas[0].size; const ExtSaveArea *esa; diff --git a/target/i386/cpu.h b/target/i386/cpu.h index d4124973ce..9327353fff 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -2071,6 +2071,8 @@ void cpu_x86_fsave(CPUX86State *s, target_ulong ptr, int data32); void cpu_x86_frstor(CPUX86State *s, target_ulong ptr, int data32); void cpu_x86_fxsave(CPUX86State *s, target_ulong ptr); void cpu_x86_fxrstor(CPUX86State *s, target_ulong ptr); +void cpu_x86_xsave(CPUX86State *s, target_ulong ptr); +void cpu_x86_xrstor(CPUX86State *s, target_ulong ptr); /* cpu.c */ void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1, @@ -2327,6 +2329,7 @@ bool cpu_is_bsp(X86CPU *cpu); void x86_cpu_xrstor_all_areas(X86CPU *cpu, const void *buf, uint32_t buflen); void x86_cpu_xsave_all_areas(X86CPU *cpu, void *buf, uint32_t buflen); +uint32_t xsave_area_size(uint64_t mask, bool compacted); void x86_update_hflags(CPUX86State* env); static inline bool hyperv_feat_enabled(X86CPU *cpu, int feat) diff --git a/target/i386/tcg/fpu_helper.c b/target/i386/tcg/fpu_helper.c index 30bc44fcf8..ad58931751 100644 --- a/target/i386/tcg/fpu_helper.c +++ b/target/i386/tcg/fpu_helper.c @@ -2502,18 +2502,6 @@ void helper_frstor(CPUX86State *env, target_ulong ptr, int data32) do_frstor(env, ptr, data32, GETPC()); } -#if defined(CONFIG_USER_ONLY) -void cpu_x86_fsave(CPUX86State *env, target_ulong ptr, int data32) -{ - do_fsave(env, ptr, data32, 0); -} - -void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32) -{ - do_frstor(env, ptr, data32, 0); -} -#endif - #define XO(X) offsetof(X86XSaveArea, X) static void do_xsave_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra) @@ -2787,21 +2775,8 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr) do_fxrstor(env, ptr, GETPC()); } -#if defined(CONFIG_USER_ONLY) -void cpu_x86_fxsave(CPUX86State *env, target_ulong ptr) +static void do_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm, uintptr_t ra) { - do_fxsave(env, ptr, 0); -} - -void cpu_x86_fxrstor(CPUX86State *env, target_ulong ptr) -{ - do_fxrstor(env, ptr, 0); -} -#endif - -void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm) -{ - uintptr_t ra = GETPC(); uint64_t xstate_bv, xcomp_bv, reserve0; rfbm &= env->xcr0; @@ -2894,6 +2869,43 @@ void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm) #undef XO +void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm) +{ + do_xrstor(env, ptr, rfbm, GETPC()); +} + +#if defined(CONFIG_USER_ONLY) +void cpu_x86_fsave(CPUX86State *env, target_ulong ptr, int data32) +{ + do_fsave(env, ptr, data32, 0); +} + +void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32) +{ + do_frstor(env, ptr, data32, 0); +} + +void cpu_x86_fxsave(CPUX86State *env, target_ulong ptr) +{ + do_fxsave(env, ptr, 0); +} + +void cpu_x86_fxrstor(CPUX86State *env, target_ulong ptr) +{ + do_fxrstor(env, ptr, 0); +} + +void cpu_x86_xsave(CPUX86State *env, target_ulong ptr) +{ + do_xsave(env, ptr, -1, get_xinuse(env), -1, 0); +} + +void cpu_x86_xrstor(CPUX86State *env, target_ulong ptr) +{ + do_xrstor(env, ptr, -1, 0); +} +#endif + uint64_t helper_xgetbv(CPUX86State *env, uint32_t ecx) { /* The OS must have enabled XSAVE. */