diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 423fb1936f..f99b0becd8 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -77,86 +77,12 @@ do { } while (0) #endif -#define KVM_MSI_HASHTAB_SIZE 256 - struct KVMParkedVcpu { unsigned long vcpu_id; int kvm_fd; QLIST_ENTRY(KVMParkedVcpu) node; }; -enum KVMDirtyRingReaperState { - KVM_DIRTY_RING_REAPER_NONE = 0, - /* The reaper is sleeping */ - KVM_DIRTY_RING_REAPER_WAIT, - /* The reaper is reaping for dirty pages */ - KVM_DIRTY_RING_REAPER_REAPING, -}; - -/* - * KVM reaper instance, responsible for collecting the KVM dirty bits - * via the dirty ring. - */ -struct KVMDirtyRingReaper { - /* The reaper thread */ - QemuThread reaper_thr; - volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ - volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ -}; - -struct KVMState -{ - AccelState parent_obj; - - int nr_slots; - int fd; - int vmfd; - int coalesced_mmio; - int coalesced_pio; - struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; - bool coalesced_flush_in_progress; - int vcpu_events; - int robust_singlestep; - int debugregs; -#ifdef KVM_CAP_SET_GUEST_DEBUG - QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; -#endif - int max_nested_state_len; - int many_ioeventfds; - int intx_set_mask; - int kvm_shadow_mem; - bool kernel_irqchip_allowed; - bool kernel_irqchip_required; - OnOffAuto kernel_irqchip_split; - bool sync_mmu; - uint64_t manual_dirty_log_protect; - /* The man page (and posix) say ioctl numbers are signed int, but - * they're not. Linux, glibc and *BSD all treat ioctl numbers as - * unsigned, and treating them as signed here can break things */ - unsigned irq_set_ioctl; - unsigned int sigmask_len; - GHashTable *gsimap; -#ifdef KVM_CAP_IRQ_ROUTING - struct kvm_irq_routing *irq_routes; - int nr_allocated_irq_routes; - unsigned long *used_gsi_bitmap; - unsigned int gsi_count; - QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; -#endif - KVMMemoryListener memory_listener; - QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; - - /* For "info mtree -f" to tell if an MR is registered in KVM */ - int nr_as; - struct KVMAs { - KVMMemoryListener *ml; - AddressSpace *as; - } *as; - uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ - uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ - struct KVMDirtyRingReaper reaper; -}; - KVMState *kvm_state; bool kvm_kernel_irqchip; bool kvm_split_irqchip; @@ -3692,6 +3618,8 @@ static void kvm_accel_instance_init(Object *obj) s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; /* KVM dirty ring is by default off */ s->kvm_dirty_ring_size = 0; + s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; + s->notify_window = 0; } /** @@ -3731,6 +3659,8 @@ static void kvm_accel_class_init(ObjectClass *oc, void *data) NULL, NULL); object_class_property_set_description(oc, "dirty-ring-size", "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); + + kvm_arch_accel_class_init(oc); } static const TypeInfo kvm_accel_type = { diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c index 399e1787ea..e493c28814 100644 --- a/hw/scsi/scsi-disk.c +++ b/hw/scsi/scsi-disk.c @@ -2544,6 +2544,7 @@ static void scsi_cd_realize(SCSIDevice *dev, Error **errp) SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev); AioContext *ctx; int ret; + uint32_t blocksize = 2048; if (!dev->conf.blk) { /* Anonymous BlockBackend for an empty drive. As we put it into @@ -2553,9 +2554,13 @@ static void scsi_cd_realize(SCSIDevice *dev, Error **errp) assert(ret == 0); } + if (dev->conf.physical_block_size != 0) { + blocksize = dev->conf.physical_block_size; + } + ctx = blk_get_aio_context(dev->conf.blk); aio_context_acquire(ctx); - s->qdev.blocksize = 2048; + s->qdev.blocksize = blocksize; s->qdev.type = TYPE_ROM; s->features |= 1 << SCSI_DISK_F_REMOVABLE; if (!s->product) { diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h index 790d35ef78..e9a97eda8c 100644 --- a/include/sysemu/kvm.h +++ b/include/sysemu/kvm.h @@ -349,6 +349,8 @@ bool kvm_device_supported(int vmfd, uint64_t type); extern const KVMCapabilityInfo kvm_arch_required_capabilities[]; +void kvm_arch_accel_class_init(ObjectClass *oc); + void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run); MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run); diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h index 1f5487d9b7..3b4adcdc10 100644 --- a/include/sysemu/kvm_int.h +++ b/include/sysemu/kvm_int.h @@ -10,6 +10,7 @@ #define QEMU_KVM_INT_H #include "exec/memory.h" +#include "qapi/qapi-types-common.h" #include "qemu/accel.h" #include "sysemu/kvm.h" @@ -36,6 +37,81 @@ typedef struct KVMMemoryListener { int as_id; } KVMMemoryListener; +#define KVM_MSI_HASHTAB_SIZE 256 + +enum KVMDirtyRingReaperState { + KVM_DIRTY_RING_REAPER_NONE = 0, + /* The reaper is sleeping */ + KVM_DIRTY_RING_REAPER_WAIT, + /* The reaper is reaping for dirty pages */ + KVM_DIRTY_RING_REAPER_REAPING, +}; + +/* + * KVM reaper instance, responsible for collecting the KVM dirty bits + * via the dirty ring. + */ +struct KVMDirtyRingReaper { + /* The reaper thread */ + QemuThread reaper_thr; + volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ + volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ +}; +struct KVMState +{ + AccelState parent_obj; + + int nr_slots; + int fd; + int vmfd; + int coalesced_mmio; + int coalesced_pio; + struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; + bool coalesced_flush_in_progress; + int vcpu_events; + int robust_singlestep; + int debugregs; +#ifdef KVM_CAP_SET_GUEST_DEBUG + QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; +#endif + int max_nested_state_len; + int many_ioeventfds; + int intx_set_mask; + int kvm_shadow_mem; + bool kernel_irqchip_allowed; + bool kernel_irqchip_required; + OnOffAuto kernel_irqchip_split; + bool sync_mmu; + uint64_t manual_dirty_log_protect; + /* The man page (and posix) say ioctl numbers are signed int, but + * they're not. Linux, glibc and *BSD all treat ioctl numbers as + * unsigned, and treating them as signed here can break things */ + unsigned irq_set_ioctl; + unsigned int sigmask_len; + GHashTable *gsimap; +#ifdef KVM_CAP_IRQ_ROUTING + struct kvm_irq_routing *irq_routes; + int nr_allocated_irq_routes; + unsigned long *used_gsi_bitmap; + unsigned int gsi_count; + QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; +#endif + KVMMemoryListener memory_listener; + QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; + + /* For "info mtree -f" to tell if an MR is registered in KVM */ + int nr_as; + struct KVMAs { + KVMMemoryListener *ml; + AddressSpace *as; + } *as; + uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ + uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ + struct KVMDirtyRingReaper reaper; + NotifyVmexitOption notify_vmexit; + uint32_t notify_window; +}; + void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, AddressSpace *as, int as_id, const char *name); diff --git a/linux-user/i386/signal.c b/linux-user/i386/signal.c index 4372621a4d..60fa07d6f9 100644 --- a/linux-user/i386/signal.c +++ b/linux-user/i386/signal.c @@ -24,6 +24,10 @@ /* from the Linux kernel - /arch/x86/include/uapi/asm/sigcontext.h */ +#define TARGET_FP_XSTATE_MAGIC1 0x46505853U /* FPXS */ +#define TARGET_FP_XSTATE_MAGIC2 0x46505845U /* FPXE */ +#define TARGET_FP_XSTATE_MAGIC2_SIZE 4 + struct target_fpreg { uint16_t significand[4]; uint16_t exponent; @@ -39,6 +43,35 @@ struct target_xmmreg { uint32_t element[4]; }; +struct target_fpx_sw_bytes { + uint32_t magic1; + uint32_t extended_size; + uint64_t xfeatures; + uint32_t xstate_size; + uint32_t reserved[7]; +}; +QEMU_BUILD_BUG_ON(sizeof(struct target_fpx_sw_bytes) != 12*4); + +struct target_fpstate_fxsave { + /* FXSAVE format */ + uint16_t cw; + uint16_t sw; + uint16_t twd; + uint16_t fop; + uint64_t rip; + uint64_t rdp; + uint32_t mxcsr; + uint32_t mxcsr_mask; + uint32_t st_space[32]; + uint32_t xmm_space[64]; + uint32_t hw_reserved[12]; + struct target_fpx_sw_bytes sw_reserved; + uint8_t xfeatures[]; +}; +#define TARGET_FXSAVE_SIZE sizeof(struct target_fpstate_fxsave) +QEMU_BUILD_BUG_ON(TARGET_FXSAVE_SIZE != 512); +QEMU_BUILD_BUG_ON(offsetof(struct target_fpstate_fxsave, sw_reserved) != 464); + struct target_fpstate_32 { /* Regular FPU environment */ uint32_t cw; @@ -51,35 +84,21 @@ struct target_fpstate_32 { struct target_fpreg st[8]; uint16_t status; uint16_t magic; /* 0xffff = regular FPU data only */ - - /* FXSR FPU environment */ - uint32_t _fxsr_env[6]; /* FXSR FPU env is ignored */ - uint32_t mxcsr; - uint32_t reserved; - struct target_fpxreg fxsr_st[8]; /* FXSR FPU reg data is ignored */ - struct target_xmmreg xmm[8]; - uint32_t padding[56]; + struct target_fpstate_fxsave fxsave; }; -struct target_fpstate_64 { - /* FXSAVE format */ - uint16_t cw; - uint16_t sw; - uint16_t twd; - uint16_t fop; - uint64_t rip; - uint64_t rdp; - uint32_t mxcsr; - uint32_t mxcsr_mask; - uint32_t st_space[32]; - uint32_t xmm_space[64]; - uint32_t reserved[24]; -}; +/* + * For simplicity, setup_frame aligns struct target_fpstate_32 to + * 16 bytes, so ensure that the FXSAVE area is also aligned. + */ +QEMU_BUILD_BUG_ON(offsetof(struct target_fpstate_32, fxsave) & 15); #ifndef TARGET_X86_64 # define target_fpstate target_fpstate_32 +# define TARGET_FPSTATE_FXSAVE_OFFSET offsetof(struct target_fpstate_32, fxsave) #else -# define target_fpstate target_fpstate_64 +# define target_fpstate target_fpstate_fxsave +# define TARGET_FPSTATE_FXSAVE_OFFSET 0 #endif struct target_sigcontext_32 { @@ -163,10 +182,25 @@ struct sigframe { abi_ulong pretcode; int sig; struct target_sigcontext sc; - struct target_fpstate fpstate; + /* + * The actual fpstate is placed after retcode[] below, to make + * room for the variable-sized xsave data. The older unused fpstate + * has to be kept to avoid changing the offset of extramask[], which + * is part of the ABI. + */ + struct target_fpstate fpstate_unused; abi_ulong extramask[TARGET_NSIG_WORDS-1]; char retcode[8]; + + /* + * This field will be 16-byte aligned in memory. Applying QEMU_ALIGNED + * to it ensures that the base of the frame has an appropriate alignment + * too. + */ + struct target_fpstate fpstate QEMU_ALIGNED(8); }; +#define TARGET_SIGFRAME_FXSAVE_OFFSET ( \ + offsetof(struct sigframe, fpstate) + TARGET_FPSTATE_FXSAVE_OFFSET) struct rt_sigframe { abi_ulong pretcode; @@ -175,26 +209,62 @@ struct rt_sigframe { abi_ulong puc; struct target_siginfo info; struct target_ucontext uc; - struct target_fpstate fpstate; char retcode[8]; + struct target_fpstate fpstate QEMU_ALIGNED(8); }; - +#define TARGET_RT_SIGFRAME_FXSAVE_OFFSET ( \ + offsetof(struct rt_sigframe, fpstate) + TARGET_FPSTATE_FXSAVE_OFFSET) #else struct rt_sigframe { abi_ulong pretcode; struct target_ucontext uc; struct target_siginfo info; - struct target_fpstate fpstate; + struct target_fpstate fpstate QEMU_ALIGNED(16); }; - +#define TARGET_RT_SIGFRAME_FXSAVE_OFFSET ( \ + offsetof(struct rt_sigframe, fpstate) + TARGET_FPSTATE_FXSAVE_OFFSET) #endif /* * Set up a signal frame. */ -/* XXX: save x87 state */ +static void xsave_sigcontext(CPUX86State *env, struct target_fpstate_fxsave *fxsave, + abi_ulong fxsave_addr) +{ + if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) { + /* fxsave_addr must be 16 byte aligned for fxsave */ + assert(!(fxsave_addr & 0xf)); + + cpu_x86_fxsave(env, fxsave_addr); + __put_user(0, &fxsave->sw_reserved.magic1); + } else { + uint32_t xstate_size = xsave_area_size(env->xcr0, false); + uint32_t xfeatures_size = xstate_size - TARGET_FXSAVE_SIZE; + + /* + * extended_size is the offset from fpstate_addr to right after the end + * of the extended save states. On 32-bit that includes the legacy + * FSAVE area. + */ + uint32_t extended_size = TARGET_FPSTATE_FXSAVE_OFFSET + + xstate_size + TARGET_FP_XSTATE_MAGIC2_SIZE; + + /* fxsave_addr must be 64 byte aligned for xsave */ + assert(!(fxsave_addr & 0x3f)); + + /* Zero the header, XSAVE *adds* features to an existing save state. */ + memset(fxsave->xfeatures, 0, 64); + cpu_x86_xsave(env, fxsave_addr); + __put_user(TARGET_FP_XSTATE_MAGIC1, &fxsave->sw_reserved.magic1); + __put_user(extended_size, &fxsave->sw_reserved.extended_size); + __put_user(env->xcr0, &fxsave->sw_reserved.xfeatures); + __put_user(xstate_size, &fxsave->sw_reserved.xstate_size); + __put_user(TARGET_FP_XSTATE_MAGIC2, (uint32_t *) &fxsave->xfeatures[xfeatures_size]); + } +} + static void setup_sigcontext(struct target_sigcontext *sc, struct target_fpstate *fpstate, CPUX86State *env, abi_ulong mask, abi_ulong fpstate_addr) @@ -226,13 +296,14 @@ static void setup_sigcontext(struct target_sigcontext *sc, cpu_x86_fsave(env, fpstate_addr, 1); fpstate->status = fpstate->sw; - magic = 0xffff; + if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { + magic = 0xffff; + } else { + xsave_sigcontext(env, &fpstate->fxsave, + fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); + magic = 0; + } __put_user(magic, &fpstate->magic); - __put_user(fpstate_addr, &sc->fpstate); - - /* non-iBCS2 extensions.. */ - __put_user(mask, &sc->oldmask); - __put_user(env->cr[2], &sc->cr2); #else __put_user(env->regs[R_EDI], &sc->rdi); __put_user(env->regs[R_ESI], &sc->rsi); @@ -262,15 +333,14 @@ static void setup_sigcontext(struct target_sigcontext *sc, __put_user((uint16_t)0, &sc->fs); __put_user(env->segs[R_SS].selector, &sc->ss); + xsave_sigcontext(env, fpstate, fpstate_addr); +#endif + + __put_user(fpstate_addr, &sc->fpstate); + + /* non-iBCS2 extensions.. */ __put_user(mask, &sc->oldmask); __put_user(env->cr[2], &sc->cr2); - - /* fpstate_addr must be 16 byte aligned for fxsave */ - assert(!(fpstate_addr & 0xf)); - - cpu_x86_fxsave(env, fpstate_addr); - __put_user(fpstate_addr, &sc->fpstate); -#endif } /* @@ -278,7 +348,7 @@ static void setup_sigcontext(struct target_sigcontext *sc, */ static inline abi_ulong -get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t frame_size) +get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t fxsave_offset) { unsigned long esp; @@ -302,11 +372,15 @@ get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t frame_size) #endif } -#ifndef TARGET_X86_64 - return (esp - frame_size) & -8ul; -#else - return ((esp - frame_size) & (~15ul)) - 8; -#endif + if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { + return (esp - (fxsave_offset + TARGET_FXSAVE_SIZE)) & -8ul; + } else if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) { + return ((esp - TARGET_FXSAVE_SIZE) & -16ul) - fxsave_offset; + } else { + size_t xstate_size = + xsave_area_size(env->xcr0, false) + TARGET_FP_XSTATE_MAGIC2_SIZE; + return ((esp - xstate_size) & -64ul) - fxsave_offset; + } } #ifndef TARGET_X86_64 @@ -334,7 +408,7 @@ void setup_frame(int sig, struct target_sigaction *ka, struct sigframe *frame; int i; - frame_addr = get_sigframe(ka, env, sizeof(*frame)); + frame_addr = get_sigframe(ka, env, TARGET_SIGFRAME_FXSAVE_OFFSET); trace_user_setup_frame(env, frame_addr); if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) @@ -390,7 +464,7 @@ void setup_rt_frame(int sig, struct target_sigaction *ka, struct rt_sigframe *frame; int i; - frame_addr = get_sigframe(ka, env, sizeof(*frame)); + frame_addr = get_sigframe(ka, env, TARGET_RT_SIGFRAME_FXSAVE_OFFSET); trace_user_setup_rt_frame(env, frame_addr); if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) @@ -409,7 +483,11 @@ void setup_rt_frame(int sig, struct target_sigaction *ka, } /* Create the ucontext. */ - __put_user(0, &frame->uc.tuc_flags); + if (env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) { + __put_user(1, &frame->uc.tuc_flags); + } else { + __put_user(0, &frame->uc.tuc_flags); + } __put_user(0, &frame->uc.tuc_link); target_save_altstack(&frame->uc.tuc_stack, env); setup_sigcontext(&frame->uc.tuc_mcontext, &frame->fpstate, env, @@ -463,10 +541,37 @@ give_sigsegv: force_sigsegv(sig); } +static int xrstor_sigcontext(CPUX86State *env, struct target_fpstate_fxsave *fxsave, + abi_ulong fxsave_addr) +{ + if (env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) { + uint32_t extended_size = tswapl(fxsave->sw_reserved.extended_size); + uint32_t xstate_size = tswapl(fxsave->sw_reserved.xstate_size); + uint32_t xfeatures_size = xstate_size - TARGET_FXSAVE_SIZE; + + /* Linux checks MAGIC2 using xstate_size, not extended_size. */ + if (tswapl(fxsave->sw_reserved.magic1) == TARGET_FP_XSTATE_MAGIC1 && + extended_size >= TARGET_FPSTATE_FXSAVE_OFFSET + xstate_size + TARGET_FP_XSTATE_MAGIC2_SIZE) { + if (!access_ok(env_cpu(env), VERIFY_READ, fxsave_addr, + extended_size - TARGET_FPSTATE_FXSAVE_OFFSET)) { + return 1; + } + if (tswapl(*(uint32_t *) &fxsave->xfeatures[xfeatures_size]) == TARGET_FP_XSTATE_MAGIC2) { + cpu_x86_xrstor(env, fxsave_addr); + return 0; + } + } + /* fall through to fxrstor */ + } + + cpu_x86_fxrstor(env, fxsave_addr); + return 0; +} + static int restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc) { - unsigned int err = 0; + int err = 1; abi_ulong fpstate_addr; unsigned int tmpflags; @@ -517,20 +622,28 @@ restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc) fpstate_addr = tswapl(sc->fpstate); if (fpstate_addr != 0) { - if (!access_ok(env_cpu(env), VERIFY_READ, fpstate_addr, - sizeof(struct target_fpstate))) { - goto badframe; + struct target_fpstate *fpstate; + if (!lock_user_struct(VERIFY_READ, fpstate, fpstate_addr, + sizeof(struct target_fpstate))) { + return err; } #ifndef TARGET_X86_64 - cpu_x86_frstor(env, fpstate_addr, 1); + if (!(env->features[FEAT_1_EDX] & CPUID_FXSR)) { + cpu_x86_frstor(env, fpstate_addr, 1); + err = 0; + } else { + err = xrstor_sigcontext(env, &fpstate->fxsave, + fpstate_addr + TARGET_FPSTATE_FXSAVE_OFFSET); + } #else - cpu_x86_fxrstor(env, fpstate_addr); + err = xrstor_sigcontext(env, fpstate, fpstate_addr); #endif + unlock_user_struct(fpstate, fpstate_addr, 0); + } else { + err = 0; } return err; -badframe: - return 1; } /* Note: there is no sigreturn on x86_64, there is only rt_sigreturn */ diff --git a/qapi/run-state.json b/qapi/run-state.json index 9273ea6516..49989d30e6 100644 --- a/qapi/run-state.json +++ b/qapi/run-state.json @@ -643,3 +643,20 @@ { 'struct': 'MemoryFailureFlags', 'data': { 'action-required': 'bool', 'recursive': 'bool'} } + +## +# @NotifyVmexitOption: +# +# An enumeration of the options specified when enabling notify VM exit +# +# @run: enable the feature, do nothing and continue if the notify VM exit happens. +# +# @internal-error: enable the feature, raise a internal error if the notify +# VM exit happens. +# +# @disable: disable the feature. +# +# Since: 7.2 +## +{ 'enum': 'NotifyVmexitOption', + 'data': [ 'run', 'internal-error', 'disable' ] } \ No newline at end of file diff --git a/qemu-options.hx b/qemu-options.hx index c0bb74655c..afb34703fe 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -191,6 +191,7 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel, " split-wx=on|off (enable TCG split w^x mapping)\n" " tb-size=n (TCG translation block cache size)\n" " dirty-ring-size=n (KVM dirty ring GFN count, default 0)\n" + " notify-vmexit=run|internal-error|disable,notify-window=n (enable notify VM exit and set notify window, x86 only)\n" " thread=single|multi (enable multi-threaded TCG)\n", QEMU_ARCH_ALL) SRST ``-accel name[,prop=value[,...]]`` @@ -242,6 +243,16 @@ SRST is disabled (dirty-ring-size=0). When enabled, KVM will instead record dirty pages in a bitmap. + ``notify-vmexit=run|internal-error|disable,notify-window=n`` + Enables or disables notify VM exit support on x86 host and specify + the corresponding notify window to trigger the VM exit if enabled. + ``run`` option enables the feature. It does nothing and continue + if the exit happens. ``internal-error`` option enables the feature. + It raises a internal error. ``disable`` option doesn't enable the feature. + This feature can mitigate the CPU stuck issue due to event windows don't + open up for a specified of time (i.e. notify-window). + Default: notify-vmexit=run,notify-window=0. + ERST DEF("smp", HAS_ARG, QEMU_OPTION_smp, diff --git a/target/arm/kvm.c b/target/arm/kvm.c index 1e4de9b42e..f022c644d2 100644 --- a/target/arm/kvm.c +++ b/target/arm/kvm.c @@ -1058,3 +1058,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/i386/cpu-param.h b/target/i386/cpu-param.h index 9740bd7abd..1e79389761 100644 --- a/target/i386/cpu-param.h +++ b/target/i386/cpu-param.h @@ -25,4 +25,8 @@ #define TARGET_PAGE_BITS 12 #define NB_MMU_MODES 3 +#ifndef CONFIG_USER_ONLY +# define TARGET_TB_PCREL 1 +#endif + #endif diff --git a/target/i386/cpu.c b/target/i386/cpu.c index ad623d91e4..8a11470507 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -1467,7 +1467,7 @@ ExtSaveArea x86_ext_save_areas[XSAVE_STATE_AREA_COUNT] = { }, }; -static uint32_t xsave_area_size(uint64_t mask, bool compacted) +uint32_t xsave_area_size(uint64_t mask, bool compacted) { uint64_t ret = x86_ext_save_areas[0].size; const ExtSaveArea *esa; @@ -6017,6 +6017,7 @@ static void x86_cpu_reset(DeviceState *dev) env->exception_has_payload = false; env->exception_payload = 0; env->nmi_injected = false; + env->triple_fault_pending = false; #if !defined(CONFIG_USER_ONLY) /* We hard-wire the BSP to the first CPU. */ apic_designate_bsp(cpu->apic_state, s->cpu_index == 0); diff --git a/target/i386/cpu.h b/target/i386/cpu.h index b75108d6a3..7edf5dfac3 100644 --- a/target/i386/cpu.h +++ b/target/i386/cpu.h @@ -1739,6 +1739,7 @@ typedef struct CPUArchState { uint8_t has_error_code; uint8_t exception_has_payload; uint64_t exception_payload; + uint8_t triple_fault_pending; uint32_t ins_len; uint32_t sipi_vector; bool tsc_valid; @@ -2070,6 +2071,8 @@ void cpu_x86_fsave(CPUX86State *s, target_ulong ptr, int data32); void cpu_x86_frstor(CPUX86State *s, target_ulong ptr, int data32); void cpu_x86_fxsave(CPUX86State *s, target_ulong ptr); void cpu_x86_fxrstor(CPUX86State *s, target_ulong ptr); +void cpu_x86_xsave(CPUX86State *s, target_ulong ptr); +void cpu_x86_xrstor(CPUX86State *s, target_ulong ptr); /* cpu.c */ void x86_cpu_vendor_words2str(char *dst, uint32_t vendor1, @@ -2326,6 +2329,7 @@ bool cpu_is_bsp(X86CPU *cpu); void x86_cpu_xrstor_all_areas(X86CPU *cpu, const void *buf, uint32_t buflen); void x86_cpu_xsave_all_areas(X86CPU *cpu, void *buf, uint32_t buflen); +uint32_t xsave_area_size(uint64_t mask, bool compacted); void x86_update_hflags(CPUX86State* env); static inline bool hyperv_feat_enabled(X86CPU *cpu, int feat) diff --git a/target/i386/helper.h b/target/i386/helper.h index ac3b4d1ee3..39a3c24182 100644 --- a/target/i386/helper.h +++ b/target/i386/helper.h @@ -37,7 +37,7 @@ DEF_HELPER_2(lldt, void, env, int) DEF_HELPER_2(ltr, void, env, int) DEF_HELPER_3(load_seg, void, env, int, int) DEF_HELPER_4(ljmp_protected, void, env, int, tl, tl) -DEF_HELPER_5(lcall_real, void, env, int, tl, int, int) +DEF_HELPER_5(lcall_real, void, env, i32, i32, int, i32) DEF_HELPER_5(lcall_protected, void, env, int, tl, int, tl) DEF_HELPER_2(iret_real, void, env, int) DEF_HELPER_3(iret_protected, void, env, int, int) diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c index f2a96492ce..bed6c00f2c 100644 --- a/target/i386/kvm/kvm.c +++ b/target/i386/kvm/kvm.c @@ -15,6 +15,7 @@ #include "qemu/osdep.h" #include "qapi/qapi-events-run-state.h" #include "qapi/error.h" +#include "qapi/visitor.h" #include #include #include @@ -132,6 +133,7 @@ static int has_xcrs; static int has_pit_state2; static int has_sregs2; static int has_exception_payload; +static int has_triple_fault_event; static bool has_msr_mcg_ext_ctl; @@ -139,6 +141,8 @@ static struct kvm_cpuid2 *cpuid_cache; static struct kvm_cpuid2 *hv_cpuid_cache; static struct kvm_msr_list *kvm_feature_msrs; +static KVMMSRHandlers msr_handlers[KVM_MSR_FILTER_MAX_RANGES]; + #define BUS_LOCK_SLICE_TIME 1000000000ULL /* ns */ static RateLimit bus_lock_ratelimit_ctrl; static int kvm_get_one_msr(X86CPU *cpu, int index, uint64_t *value); @@ -2397,6 +2401,17 @@ static int kvm_get_supported_msrs(KVMState *s) return ret; } +static bool kvm_rdmsr_core_thread_count(X86CPU *cpu, uint32_t msr, + uint64_t *val) +{ + CPUState *cs = CPU(cpu); + + *val = cs->nr_threads * cs->nr_cores; /* thread count, bits 15..0 */ + *val |= ((uint32_t)cs->nr_cores << 16); /* core count, bits 31..16 */ + + return true; +} + static Notifier smram_machine_done; static KVMMemoryListener smram_listener; static AddressSpace smram_address_space; @@ -2479,6 +2494,16 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + has_triple_fault_event = kvm_check_extension(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT); + if (has_triple_fault_event) { + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT, 0, true); + if (ret < 0) { + error_report("kvm: Failed to enable triple fault event cap: %s", + strerror(-ret)); + return ret; + } + } + ret = kvm_get_supported_msrs(s); if (ret < 0) { return ret; @@ -2584,6 +2609,40 @@ int kvm_arch_init(MachineState *ms, KVMState *s) } } + if (s->notify_vmexit != NOTIFY_VMEXIT_OPTION_DISABLE && + kvm_check_extension(s, KVM_CAP_X86_NOTIFY_VMEXIT)) { + uint64_t notify_window_flags = + ((uint64_t)s->notify_window << 32) | + KVM_X86_NOTIFY_VMEXIT_ENABLED | + KVM_X86_NOTIFY_VMEXIT_USER; + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_NOTIFY_VMEXIT, 0, + notify_window_flags); + if (ret < 0) { + error_report("kvm: Failed to enable notify vmexit cap: %s", + strerror(-ret)); + return ret; + } + } + if (kvm_vm_check_extension(s, KVM_CAP_X86_USER_SPACE_MSR)) { + bool r; + + ret = kvm_vm_enable_cap(s, KVM_CAP_X86_USER_SPACE_MSR, 0, + KVM_MSR_EXIT_REASON_FILTER); + if (ret) { + error_report("Could not enable user space MSRs: %s", + strerror(-ret)); + exit(1); + } + + r = kvm_filter_msr(s, MSR_CORE_THREAD_COUNT, + kvm_rdmsr_core_thread_count, NULL); + if (!r) { + error_report("Could not install MSR_CORE_THREAD_COUNT handler: %s", + strerror(-ret)); + exit(1); + } + } + return 0; } @@ -4295,6 +4354,11 @@ static int kvm_put_vcpu_events(X86CPU *cpu, int level) } } + if (has_triple_fault_event) { + events.flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT; + events.triple_fault.pending = env->triple_fault_pending; + } + return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events); } @@ -4364,6 +4428,10 @@ static int kvm_get_vcpu_events(X86CPU *cpu) } } + if (events.flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) { + env->triple_fault_pending = events.triple_fault.pending; + } + env->sipi_vector = events.sipi_vector; return 0; @@ -5073,6 +5141,108 @@ void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg) } } +static bool kvm_install_msr_filters(KVMState *s) +{ + uint64_t zero = 0; + struct kvm_msr_filter filter = { + .flags = KVM_MSR_FILTER_DEFAULT_ALLOW, + }; + int r, i, j = 0; + + for (i = 0; i < KVM_MSR_FILTER_MAX_RANGES; i++) { + KVMMSRHandlers *handler = &msr_handlers[i]; + if (handler->msr) { + struct kvm_msr_filter_range *range = &filter.ranges[j++]; + + *range = (struct kvm_msr_filter_range) { + .flags = 0, + .nmsrs = 1, + .base = handler->msr, + .bitmap = (__u8 *)&zero, + }; + + if (handler->rdmsr) { + range->flags |= KVM_MSR_FILTER_READ; + } + + if (handler->wrmsr) { + range->flags |= KVM_MSR_FILTER_WRITE; + } + } + } + + r = kvm_vm_ioctl(s, KVM_X86_SET_MSR_FILTER, &filter); + if (r) { + return false; + } + + return true; +} + +bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr, + QEMUWRMSRHandler *wrmsr) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) { + if (!msr_handlers[i].msr) { + msr_handlers[i] = (KVMMSRHandlers) { + .msr = msr, + .rdmsr = rdmsr, + .wrmsr = wrmsr, + }; + + if (!kvm_install_msr_filters(s)) { + msr_handlers[i] = (KVMMSRHandlers) { }; + return false; + } + + return true; + } + } + + return false; +} + +static int kvm_handle_rdmsr(X86CPU *cpu, struct kvm_run *run) +{ + int i; + bool r; + + for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) { + KVMMSRHandlers *handler = &msr_handlers[i]; + if (run->msr.index == handler->msr) { + if (handler->rdmsr) { + r = handler->rdmsr(cpu, handler->msr, + (uint64_t *)&run->msr.data); + run->msr.error = r ? 0 : 1; + return 0; + } + } + } + + assert(false); +} + +static int kvm_handle_wrmsr(X86CPU *cpu, struct kvm_run *run) +{ + int i; + bool r; + + for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) { + KVMMSRHandlers *handler = &msr_handlers[i]; + if (run->msr.index == handler->msr) { + if (handler->wrmsr) { + r = handler->wrmsr(cpu, handler->msr, run->msr.data); + run->msr.error = r ? 0 : 1; + return 0; + } + } + } + + assert(false); +} + static bool has_sgx_provisioning; static bool __kvm_enable_sgx_provisioning(KVMState *s) @@ -5117,6 +5287,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) X86CPU *cpu = X86_CPU(cs); uint64_t code; int ret; + bool ctx_invalid; + char str[256]; + KVMState *state; switch (run->exit_reason) { case KVM_EXIT_HLT: @@ -5172,6 +5345,31 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) /* already handled in kvm_arch_post_run */ ret = 0; break; + case KVM_EXIT_NOTIFY: + ctx_invalid = !!(run->notify.flags & KVM_NOTIFY_CONTEXT_INVALID); + state = KVM_STATE(current_accel()); + sprintf(str, "Encounter a notify exit with %svalid context in" + " guest. There can be possible misbehaves in guest." + " Please have a look.", ctx_invalid ? "in" : ""); + if (ctx_invalid || + state->notify_vmexit == NOTIFY_VMEXIT_OPTION_INTERNAL_ERROR) { + warn_report("KVM internal error: %s", str); + ret = -1; + } else { + warn_report_once("KVM: %s", str); + ret = 0; + } + break; + case KVM_EXIT_X86_RDMSR: + /* We only enable MSR filtering, any other exit is bogus */ + assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER); + ret = kvm_handle_rdmsr(cpu, run); + break; + case KVM_EXIT_X86_WRMSR: + /* We only enable MSR filtering, any other exit is bogus */ + assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER); + ret = kvm_handle_wrmsr(cpu, run); + break; default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); ret = -1; @@ -5448,3 +5646,71 @@ void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask) mask &= ~BIT_ULL(bit); } } + +static int kvm_arch_get_notify_vmexit(Object *obj, Error **errp) +{ + KVMState *s = KVM_STATE(obj); + return s->notify_vmexit; +} + +static void kvm_arch_set_notify_vmexit(Object *obj, int value, Error **errp) +{ + KVMState *s = KVM_STATE(obj); + + if (s->fd != -1) { + error_setg(errp, "Cannot set properties after the accelerator has been initialized"); + return; + } + + s->notify_vmexit = value; +} + +static void kvm_arch_get_notify_window(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + uint32_t value = s->notify_window; + + visit_type_uint32(v, name, &value, errp); +} + +static void kvm_arch_set_notify_window(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + KVMState *s = KVM_STATE(obj); + Error *error = NULL; + uint32_t value; + + if (s->fd != -1) { + error_setg(errp, "Cannot set properties after the accelerator has been initialized"); + return; + } + + visit_type_uint32(v, name, &value, &error); + if (error) { + error_propagate(errp, error); + return; + } + + s->notify_window = value; +} + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ + object_class_property_add_enum(oc, "notify-vmexit", "NotifyVMexitOption", + &NotifyVmexitOption_lookup, + kvm_arch_get_notify_vmexit, + kvm_arch_set_notify_vmexit); + object_class_property_set_description(oc, "notify-vmexit", + "Enable notify VM exit"); + + object_class_property_add(oc, "notify-window", "uint32", + kvm_arch_get_notify_window, + kvm_arch_set_notify_window, + NULL, NULL); + object_class_property_set_description(oc, "notify-window", + "Clock cycles without an event window " + "after which a notification VM exit occurs"); +} diff --git a/target/i386/kvm/kvm_i386.h b/target/i386/kvm/kvm_i386.h index 4124912c20..2ed586c11b 100644 --- a/target/i386/kvm/kvm_i386.h +++ b/target/i386/kvm/kvm_i386.h @@ -54,4 +54,15 @@ uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address); bool kvm_enable_sgx_provisioning(KVMState *s); void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask); +typedef bool QEMURDMSRHandler(X86CPU *cpu, uint32_t msr, uint64_t *val); +typedef bool QEMUWRMSRHandler(X86CPU *cpu, uint32_t msr, uint64_t val); +typedef struct kvm_msr_handlers { + uint32_t msr; + QEMURDMSRHandler *rdmsr; + QEMUWRMSRHandler *wrmsr; +} KVMMSRHandlers; + +bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr, + QEMUWRMSRHandler *wrmsr); + #endif diff --git a/target/i386/machine.c b/target/i386/machine.c index cecd476e98..310b125235 100644 --- a/target/i386/machine.c +++ b/target/i386/machine.c @@ -1562,6 +1562,25 @@ static const VMStateDescription vmstate_arch_lbr = { } }; +static bool triple_fault_needed(void *opaque) +{ + X86CPU *cpu = opaque; + CPUX86State *env = &cpu->env; + + return env->triple_fault_pending; +} + +static const VMStateDescription vmstate_triple_fault = { + .name = "cpu/triple_fault", + .version_id = 1, + .minimum_version_id = 1, + .needed = triple_fault_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT8(env.triple_fault_pending, X86CPU), + VMSTATE_END_OF_LIST() + } +}; + const VMStateDescription vmstate_x86_cpu = { .name = "cpu", .version_id = 12, @@ -1706,6 +1725,7 @@ const VMStateDescription vmstate_x86_cpu = { &vmstate_amx_xtile, #endif &vmstate_arch_lbr, + &vmstate_triple_fault, NULL } }; diff --git a/target/i386/tcg/fpu_helper.c b/target/i386/tcg/fpu_helper.c index 30bc44fcf8..ad58931751 100644 --- a/target/i386/tcg/fpu_helper.c +++ b/target/i386/tcg/fpu_helper.c @@ -2502,18 +2502,6 @@ void helper_frstor(CPUX86State *env, target_ulong ptr, int data32) do_frstor(env, ptr, data32, GETPC()); } -#if defined(CONFIG_USER_ONLY) -void cpu_x86_fsave(CPUX86State *env, target_ulong ptr, int data32) -{ - do_fsave(env, ptr, data32, 0); -} - -void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32) -{ - do_frstor(env, ptr, data32, 0); -} -#endif - #define XO(X) offsetof(X86XSaveArea, X) static void do_xsave_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra) @@ -2787,21 +2775,8 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr) do_fxrstor(env, ptr, GETPC()); } -#if defined(CONFIG_USER_ONLY) -void cpu_x86_fxsave(CPUX86State *env, target_ulong ptr) +static void do_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm, uintptr_t ra) { - do_fxsave(env, ptr, 0); -} - -void cpu_x86_fxrstor(CPUX86State *env, target_ulong ptr) -{ - do_fxrstor(env, ptr, 0); -} -#endif - -void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm) -{ - uintptr_t ra = GETPC(); uint64_t xstate_bv, xcomp_bv, reserve0; rfbm &= env->xcr0; @@ -2894,6 +2869,43 @@ void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm) #undef XO +void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm) +{ + do_xrstor(env, ptr, rfbm, GETPC()); +} + +#if defined(CONFIG_USER_ONLY) +void cpu_x86_fsave(CPUX86State *env, target_ulong ptr, int data32) +{ + do_fsave(env, ptr, data32, 0); +} + +void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32) +{ + do_frstor(env, ptr, data32, 0); +} + +void cpu_x86_fxsave(CPUX86State *env, target_ulong ptr) +{ + do_fxsave(env, ptr, 0); +} + +void cpu_x86_fxrstor(CPUX86State *env, target_ulong ptr) +{ + do_fxrstor(env, ptr, 0); +} + +void cpu_x86_xsave(CPUX86State *env, target_ulong ptr) +{ + do_xsave(env, ptr, -1, get_xinuse(env), -1, 0); +} + +void cpu_x86_xrstor(CPUX86State *env, target_ulong ptr) +{ + do_xrstor(env, ptr, -1, 0); +} +#endif + uint64_t helper_xgetbv(CPUX86State *env, uint32_t ecx) { /* The OS must have enabled XSAVE. */ diff --git a/target/i386/tcg/seg_helper.c b/target/i386/tcg/seg_helper.c index bffd82923f..539189b4d1 100644 --- a/target/i386/tcg/seg_helper.c +++ b/target/i386/tcg/seg_helper.c @@ -1504,14 +1504,12 @@ void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip, } /* real mode call */ -void helper_lcall_real(CPUX86State *env, int new_cs, target_ulong new_eip1, - int shift, int next_eip) +void helper_lcall_real(CPUX86State *env, uint32_t new_cs, uint32_t new_eip, + int shift, uint32_t next_eip) { - int new_eip; uint32_t esp, esp_mask; target_ulong ssp; - new_eip = new_eip1; esp = env->regs[R_ESP]; esp_mask = get_sp_mask(env->segs[R_SS].flags); ssp = env->segs[R_SS].base; diff --git a/target/i386/tcg/sysemu/misc_helper.c b/target/i386/tcg/sysemu/misc_helper.c index 1328aa656f..e1528b7f80 100644 --- a/target/i386/tcg/sysemu/misc_helper.c +++ b/target/i386/tcg/sysemu/misc_helper.c @@ -450,6 +450,11 @@ void helper_rdmsr(CPUX86State *env) case MSR_IA32_UCODE_REV: val = x86_cpu->ucode_rev; break; + case MSR_CORE_THREAD_COUNT: { + CPUState *cs = CPU(x86_cpu); + val = (cs->nr_threads * cs->nr_cores) | (cs->nr_cores << 16); + break; + } default: if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL + diff --git a/target/i386/tcg/tcg-cpu.c b/target/i386/tcg/tcg-cpu.c index 6cf14c83ff..828244abe2 100644 --- a/target/i386/tcg/tcg-cpu.c +++ b/target/i386/tcg/tcg-cpu.c @@ -49,9 +49,11 @@ static void x86_cpu_exec_exit(CPUState *cs) static void x86_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb) { - X86CPU *cpu = X86_CPU(cs); - - cpu->env.eip = tb_pc(tb) - tb->cs_base; + /* The instruction pointer is always up to date with TARGET_TB_PCREL. */ + if (!TARGET_TB_PCREL) { + CPUX86State *env = cs->env_ptr; + env->eip = tb_pc(tb) - tb->cs_base; + } } #ifndef CONFIG_USER_ONLY diff --git a/target/i386/tcg/translate.c b/target/i386/tcg/translate.c index 44af8c107f..279a3ae999 100644 --- a/target/i386/tcg/translate.c +++ b/target/i386/tcg/translate.c @@ -64,6 +64,7 @@ /* global register indexes */ static TCGv cpu_cc_dst, cpu_cc_src, cpu_cc_src2; +static TCGv cpu_eip; static TCGv_i32 cpu_cc_op; static TCGv cpu_regs[CPU_NB_REGS]; static TCGv cpu_seg_base[6]; @@ -76,8 +77,8 @@ typedef struct DisasContext { DisasContextBase base; target_ulong pc; /* pc = eip + cs_base */ - target_ulong pc_start; /* pc at TB entry */ target_ulong cs_base; /* base of CS segment */ + target_ulong pc_save; MemOp aflag; MemOp dflag; @@ -133,6 +134,11 @@ typedef struct DisasContext { TCGOp *prev_insn_end; } DisasContext; +#define DISAS_EOB_ONLY DISAS_TARGET_0 +#define DISAS_EOB_NEXT DISAS_TARGET_1 +#define DISAS_EOB_INHIBIT_IRQ DISAS_TARGET_2 +#define DISAS_JUMP DISAS_TARGET_3 + /* The environment in which user-only runs is constrained. */ #ifdef CONFIG_USER_ONLY #define PE(S) true @@ -219,9 +225,9 @@ STUB_HELPER(wrmsr, TCGv_env env) #endif static void gen_eob(DisasContext *s); -static void gen_jr(DisasContext *s, TCGv dest); -static void gen_jmp(DisasContext *s, target_ulong eip); -static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num); +static void gen_jr(DisasContext *s); +static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num); +static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num); static void gen_op(DisasContext *s1, int op, MemOp ot, int d); static void gen_exception_gpf(DisasContext *s); @@ -475,9 +481,10 @@ static void gen_add_A0_im(DisasContext *s, int val) } } -static inline void gen_op_jmp_v(TCGv dest) +static inline void gen_op_jmp_v(DisasContext *s, TCGv dest) { - tcg_gen_st_tl(dest, cpu_env, offsetof(CPUX86State, eip)); + tcg_gen_mov_tl(cpu_eip, dest); + s->pc_save = -1; } static inline @@ -512,10 +519,84 @@ static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d) } } -static inline void gen_jmp_im(DisasContext *s, target_ulong pc) +static void gen_update_eip_cur(DisasContext *s) { - tcg_gen_movi_tl(s->tmp0, pc); - gen_op_jmp_v(s->tmp0); + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + tcg_gen_addi_tl(cpu_eip, cpu_eip, s->base.pc_next - s->pc_save); + } else { + tcg_gen_movi_tl(cpu_eip, s->base.pc_next - s->cs_base); + } + s->pc_save = s->base.pc_next; +} + +static void gen_update_eip_next(DisasContext *s) +{ + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + tcg_gen_addi_tl(cpu_eip, cpu_eip, s->pc - s->pc_save); + } else { + tcg_gen_movi_tl(cpu_eip, s->pc - s->cs_base); + } + s->pc_save = s->pc; +} + +static int cur_insn_len(DisasContext *s) +{ + return s->pc - s->base.pc_next; +} + +static TCGv_i32 cur_insn_len_i32(DisasContext *s) +{ + return tcg_constant_i32(cur_insn_len(s)); +} + +static TCGv_i32 eip_next_i32(DisasContext *s) +{ + assert(s->pc_save != -1); + /* + * This function has two users: lcall_real (always 16-bit mode), and + * iret_protected (16, 32, or 64-bit mode). IRET only uses the value + * when EFLAGS.NT is set, which is illegal in 64-bit mode, which is + * why passing a 32-bit value isn't broken. To avoid using this where + * we shouldn't, return -1 in 64-bit mode so that execution goes into + * the weeds quickly. + */ + if (CODE64(s)) { + return tcg_constant_i32(-1); + } + if (TARGET_TB_PCREL) { + TCGv_i32 ret = tcg_temp_new_i32(); + tcg_gen_trunc_tl_i32(ret, cpu_eip); + tcg_gen_addi_i32(ret, ret, s->pc - s->pc_save); + return ret; + } else { + return tcg_constant_i32(s->pc - s->cs_base); + } +} + +static TCGv eip_next_tl(DisasContext *s) +{ + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + TCGv ret = tcg_temp_new(); + tcg_gen_addi_tl(ret, cpu_eip, s->pc - s->pc_save); + return ret; + } else { + return tcg_constant_tl(s->pc - s->cs_base); + } +} + +static TCGv eip_cur_tl(DisasContext *s) +{ + assert(s->pc_save != -1); + if (TARGET_TB_PCREL) { + TCGv ret = tcg_temp_new(); + tcg_gen_addi_tl(ret, cpu_eip, s->base.pc_next - s->pc_save); + return ret; + } else { + return tcg_constant_tl(s->base.pc_next - s->cs_base); + } } /* Compute SEG:REG into A0. SEG is selected from the override segment @@ -631,20 +712,21 @@ static void gen_exts(MemOp ot, TCGv reg) gen_ext_tl(reg, reg, ot, true); } -static inline -void gen_op_jnz_ecx(DisasContext *s, MemOp size, TCGLabel *label1) +static void gen_op_j_ecx(DisasContext *s, TCGCond cond, TCGLabel *label1) { tcg_gen_mov_tl(s->tmp0, cpu_regs[R_ECX]); - gen_extu(size, s->tmp0); - tcg_gen_brcondi_tl(TCG_COND_NE, s->tmp0, 0, label1); + gen_extu(s->aflag, s->tmp0); + tcg_gen_brcondi_tl(cond, s->tmp0, 0, label1); } -static inline -void gen_op_jz_ecx(DisasContext *s, MemOp size, TCGLabel *label1) +static inline void gen_op_jz_ecx(DisasContext *s, TCGLabel *label1) { - tcg_gen_mov_tl(s->tmp0, cpu_regs[R_ECX]); - gen_extu(size, s->tmp0); - tcg_gen_brcondi_tl(TCG_COND_EQ, s->tmp0, 0, label1); + gen_op_j_ecx(s, TCG_COND_EQ, label1); +} + +static inline void gen_op_jnz_ecx(DisasContext *s, TCGLabel *label1) +{ + gen_op_j_ecx(s, TCG_COND_NE, label1); } static void gen_helper_in_func(MemOp ot, TCGv v, TCGv_i32 n) @@ -700,24 +782,21 @@ static bool gen_check_io(DisasContext *s, MemOp ot, TCGv_i32 port, gen_helper_check_io(cpu_env, port, tcg_constant_i32(1 << ot)); } if (GUEST(s)) { - target_ulong cur_eip = s->base.pc_next - s->cs_base; - target_ulong next_eip = s->pc - s->cs_base; - gen_update_cc_op(s); - gen_jmp_im(s, cur_eip); + gen_update_eip_cur(s); if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { svm_flags |= SVM_IOIO_REP_MASK; } svm_flags |= 1 << (SVM_IOIO_SIZE_SHIFT + ot); gen_helper_svm_check_io(cpu_env, port, tcg_constant_i32(svm_flags), - tcg_constant_i32(next_eip - cur_eip)); + cur_insn_len_i32(s)); } return true; #endif } -static inline void gen_movs(DisasContext *s, MemOp ot) +static void gen_movs(DisasContext *s, MemOp ot) { gen_string_movl_A0_ESI(s); gen_op_ld_v(s, ot, s->T0, s->A0); @@ -1137,18 +1216,18 @@ static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1) /* XXX: does not work with gdbstub "ice" single step - not a serious problem */ -static TCGLabel *gen_jz_ecx_string(DisasContext *s, target_ulong next_eip) +static TCGLabel *gen_jz_ecx_string(DisasContext *s) { TCGLabel *l1 = gen_new_label(); TCGLabel *l2 = gen_new_label(); - gen_op_jnz_ecx(s, s->aflag, l1); + gen_op_jnz_ecx(s, l1); gen_set_label(l2); - gen_jmp_tb(s, next_eip, 1); + gen_jmp_rel_csize(s, 0, 1); gen_set_label(l1); return l2; } -static inline void gen_stos(DisasContext *s, MemOp ot) +static void gen_stos(DisasContext *s, MemOp ot) { gen_op_mov_v_reg(s, MO_32, s->T0, R_EAX); gen_string_movl_A0_EDI(s); @@ -1157,7 +1236,7 @@ static inline void gen_stos(DisasContext *s, MemOp ot) gen_op_add_reg_T0(s, s->aflag, R_EDI); } -static inline void gen_lods(DisasContext *s, MemOp ot) +static void gen_lods(DisasContext *s, MemOp ot) { gen_string_movl_A0_ESI(s); gen_op_ld_v(s, ot, s->T0, s->A0); @@ -1166,7 +1245,7 @@ static inline void gen_lods(DisasContext *s, MemOp ot) gen_op_add_reg_T0(s, s->aflag, R_ESI); } -static inline void gen_scas(DisasContext *s, MemOp ot) +static void gen_scas(DisasContext *s, MemOp ot) { gen_string_movl_A0_EDI(s); gen_op_ld_v(s, ot, s->T1, s->A0); @@ -1175,7 +1254,7 @@ static inline void gen_scas(DisasContext *s, MemOp ot) gen_op_add_reg_T0(s, s->aflag, R_EDI); } -static inline void gen_cmps(DisasContext *s, MemOp ot) +static void gen_cmps(DisasContext *s, MemOp ot) { gen_string_movl_A0_EDI(s); gen_op_ld_v(s, ot, s->T1, s->A0); @@ -1193,17 +1272,14 @@ static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot) /* user-mode cpu should not be in IOBPT mode */ g_assert_not_reached(); #else - TCGv_i32 t_size = tcg_const_i32(1 << ot); - TCGv t_next = tcg_const_tl(s->pc - s->cs_base); - + TCGv_i32 t_size = tcg_constant_i32(1 << ot); + TCGv t_next = eip_next_tl(s); gen_helper_bpt_io(cpu_env, t_port, t_size, t_next); - tcg_temp_free_i32(t_size); - tcg_temp_free(t_next); #endif /* CONFIG_USER_ONLY */ } } -static inline void gen_ins(DisasContext *s, MemOp ot) +static void gen_ins(DisasContext *s, MemOp ot) { gen_string_movl_A0_EDI(s); /* Note: we must do this dummy write first to be restartable in @@ -1219,7 +1295,7 @@ static inline void gen_ins(DisasContext *s, MemOp ot) gen_bpt_io(s, s->tmp2_i32, ot); } -static inline void gen_outs(DisasContext *s, MemOp ot) +static void gen_outs(DisasContext *s, MemOp ot) { gen_string_movl_A0_ESI(s); gen_op_ld_v(s, ot, s->T0, s->A0); @@ -1233,42 +1309,49 @@ static inline void gen_outs(DisasContext *s, MemOp ot) gen_bpt_io(s, s->tmp2_i32, ot); } -/* same method as Valgrind : we generate jumps to current or next - instruction */ -#define GEN_REPZ(op) \ -static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, \ - target_ulong cur_eip, target_ulong next_eip) \ -{ \ - TCGLabel *l2; \ - gen_update_cc_op(s); \ - l2 = gen_jz_ecx_string(s, next_eip); \ - gen_ ## op(s, ot); \ - gen_op_add_reg_im(s, s->aflag, R_ECX, -1); \ - /* a loop would cause two single step exceptions if ECX = 1 \ - before rep string_insn */ \ - if (s->repz_opt) \ - gen_op_jz_ecx(s, s->aflag, l2); \ - gen_jmp(s, cur_eip); \ +/* Generate jumps to current or next instruction */ +static void gen_repz(DisasContext *s, MemOp ot, + void (*fn)(DisasContext *s, MemOp ot)) +{ + TCGLabel *l2; + gen_update_cc_op(s); + l2 = gen_jz_ecx_string(s); + fn(s, ot); + gen_op_add_reg_im(s, s->aflag, R_ECX, -1); + /* + * A loop would cause two single step exceptions if ECX = 1 + * before rep string_insn + */ + if (s->repz_opt) { + gen_op_jz_ecx(s, l2); + } + gen_jmp_rel_csize(s, -cur_insn_len(s), 0); } -#define GEN_REPZ2(op) \ -static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, \ - target_ulong cur_eip, \ - target_ulong next_eip, \ - int nz) \ -{ \ - TCGLabel *l2; \ - gen_update_cc_op(s); \ - l2 = gen_jz_ecx_string(s, next_eip); \ - gen_ ## op(s, ot); \ - gen_op_add_reg_im(s, s->aflag, R_ECX, -1); \ - gen_update_cc_op(s); \ - gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); \ - if (s->repz_opt) \ - gen_op_jz_ecx(s, s->aflag, l2); \ - gen_jmp(s, cur_eip); \ +#define GEN_REPZ(op) \ + static inline void gen_repz_ ## op(DisasContext *s, MemOp ot) \ + { gen_repz(s, ot, gen_##op); } + +static void gen_repz2(DisasContext *s, MemOp ot, int nz, + void (*fn)(DisasContext *s, MemOp ot)) +{ + TCGLabel *l2; + gen_update_cc_op(s); + l2 = gen_jz_ecx_string(s); + fn(s, ot); + gen_op_add_reg_im(s, s->aflag, R_ECX, -1); + gen_update_cc_op(s); + gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2); + if (s->repz_opt) { + gen_op_jz_ecx(s, l2); + } + gen_jmp_rel_csize(s, -cur_insn_len(s), 0); } +#define GEN_REPZ2(op) \ + static inline void gen_repz_ ## op(DisasContext *s, MemOp ot, int nz) \ + { gen_repz2(s, ot, nz, gen_##op); } + GEN_REPZ(movs) GEN_REPZ(stos) GEN_REPZ(lods) @@ -1333,10 +1416,10 @@ static void gen_helper_fp_arith_STN_ST0(int op, int opreg) } } -static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip) +static void gen_exception(DisasContext *s, int trapno) { gen_update_cc_op(s); - gen_jmp_im(s, cur_eip); + gen_update_eip_cur(s); gen_helper_raise_exception(cpu_env, tcg_const_i32(trapno)); s->base.is_jmp = DISAS_NORETURN; } @@ -1345,13 +1428,13 @@ static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip) the instruction is known, but it isn't allowed in the current cpu mode. */ static void gen_illegal_opcode(DisasContext *s) { - gen_exception(s, EXCP06_ILLOP, s->pc_start - s->cs_base); + gen_exception(s, EXCP06_ILLOP); } /* Generate #GP for the current instruction. */ static void gen_exception_gpf(DisasContext *s) { - gen_exception(s, EXCP0D_GPF, s->pc_start - s->cs_base); + gen_exception(s, EXCP0D_GPF); } /* Check for cpl == 0; if not, raise #GP and return false. */ @@ -2016,7 +2099,7 @@ static uint64_t advance_pc(CPUX86State *env, DisasContext *s, int num_bytes) } s->pc += num_bytes; - if (unlikely(s->pc - s->pc_start > X86_MAX_INSN_LENGTH)) { + if (unlikely(cur_insn_len(s) > X86_MAX_INSN_LENGTH)) { /* If the instruction's 16th byte is on a different page than the 1st, a * page fault on the second page wins over the general protection fault * caused by the instruction being too long. @@ -2213,7 +2296,12 @@ static TCGv gen_lea_modrm_1(DisasContext *s, AddressParts a) ea = cpu_regs[a.base]; } if (!ea) { - tcg_gen_movi_tl(s->A0, a.disp); + if (TARGET_TB_PCREL && a.base == -2) { + /* With cpu_eip ~= pc_save, the expression is pc-relative. */ + tcg_gen_addi_tl(s->A0, cpu_eip, a.disp - s->pc_save); + } else { + tcg_gen_movi_tl(s->A0, a.disp); + } ea = s->A0; } else if (a.disp != 0) { tcg_gen_addi_tl(s->A0, ea, a.disp); @@ -2346,49 +2434,14 @@ static inline int insn_const_size(MemOp ot) } } -static void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) +static void gen_jcc(DisasContext *s, int b, int diff) { - target_ulong pc = s->cs_base + eip; + TCGLabel *l1 = gen_new_label(); - if (translator_use_goto_tb(&s->base, pc)) { - /* jump to same page: we can use a direct jump */ - tcg_gen_goto_tb(tb_num); - gen_jmp_im(s, eip); - tcg_gen_exit_tb(s->base.tb, tb_num); - s->base.is_jmp = DISAS_NORETURN; - } else { - /* jump to another page */ - gen_jmp_im(s, eip); - gen_jr(s, s->tmp0); - } -} - -static inline void gen_jcc(DisasContext *s, int b, - target_ulong val, target_ulong next_eip) -{ - TCGLabel *l1, *l2; - - if (s->jmp_opt) { - l1 = gen_new_label(); - gen_jcc1(s, b, l1); - - gen_goto_tb(s, 0, next_eip); - - gen_set_label(l1); - gen_goto_tb(s, 1, val); - } else { - l1 = gen_new_label(); - l2 = gen_new_label(); - gen_jcc1(s, b, l1); - - gen_jmp_im(s, next_eip); - tcg_gen_br(l2); - - gen_set_label(l1); - gen_jmp_im(s, val); - gen_set_label(l2); - gen_eob(s); - } + gen_jcc1(s, b, l1); + gen_jmp_rel_csize(s, 0, 1); + gen_set_label(l1); + gen_jmp_rel(s, s->dflag, diff, 0); } static void gen_cmovcc1(CPUX86State *env, DisasContext *s, MemOp ot, int b, @@ -2445,13 +2498,15 @@ static void gen_movl_seg_T0(DisasContext *s, X86Seg seg_reg) because ss32 may change. For R_SS, translation must always stop as a special handling must be done to disable hardware interrupts for the next instruction */ - if (seg_reg == R_SS || (CODE32(s) && seg_reg < R_FS)) { - s->base.is_jmp = DISAS_TOO_MANY; + if (seg_reg == R_SS) { + s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; + } else if (CODE32(s) && seg_reg < R_FS) { + s->base.is_jmp = DISAS_EOB_NEXT; } } else { gen_op_movl_seg_T0_vm(s, seg_reg); if (seg_reg == R_SS) { - s->base.is_jmp = DISAS_TOO_MANY; + s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; } } } @@ -2614,7 +2669,7 @@ static void gen_unknown_opcode(CPUX86State *env, DisasContext *s) if (qemu_loglevel_mask(LOG_UNIMP)) { FILE *logfile = qemu_log_trylock(); if (logfile) { - target_ulong pc = s->pc_start, end = s->pc; + target_ulong pc = s->base.pc_next, end = s->pc; fprintf(logfile, "ILLOPC: " TARGET_FMT_lx ":", pc); for (; pc < end; ++pc) { @@ -2628,13 +2683,12 @@ static void gen_unknown_opcode(CPUX86State *env, DisasContext *s) /* an interrupt is different from an exception because of the privilege checks */ -static void gen_interrupt(DisasContext *s, int intno, - target_ulong cur_eip, target_ulong next_eip) +static void gen_interrupt(DisasContext *s, int intno) { gen_update_cc_op(s); - gen_jmp_im(s, cur_eip); - gen_helper_raise_interrupt(cpu_env, tcg_const_i32(intno), - tcg_const_i32(next_eip - cur_eip)); + gen_update_eip_cur(s); + gen_helper_raise_interrupt(cpu_env, tcg_constant_i32(intno), + cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; } @@ -2727,28 +2781,74 @@ static void gen_eob(DisasContext *s) } /* Jump to register */ -static void gen_jr(DisasContext *s, TCGv dest) +static void gen_jr(DisasContext *s) { do_gen_eob_worker(s, false, false, true); } -/* generate a jump to eip. No segment change must happen before as a - direct call to the next block may occur */ -static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num) +/* Jump to eip+diff, truncating the result to OT. */ +static void gen_jmp_rel(DisasContext *s, MemOp ot, int diff, int tb_num) { + bool use_goto_tb = s->jmp_opt; + target_ulong mask = -1; + target_ulong new_pc = s->pc + diff; + target_ulong new_eip = new_pc - s->cs_base; + + /* In 64-bit mode, operand size is fixed at 64 bits. */ + if (!CODE64(s)) { + if (ot == MO_16) { + mask = 0xffff; + if (TARGET_TB_PCREL && CODE32(s)) { + use_goto_tb = false; + } + } else { + mask = 0xffffffff; + } + } + new_eip &= mask; + gen_update_cc_op(s); set_cc_op(s, CC_OP_DYNAMIC); - if (s->jmp_opt) { - gen_goto_tb(s, tb_num, eip); + + if (TARGET_TB_PCREL) { + tcg_gen_addi_tl(cpu_eip, cpu_eip, new_pc - s->pc_save); + /* + * If we can prove the branch does not leave the page and we have + * no extra masking to apply (data16 branch in code32, see above), + * then we have also proven that the addition does not wrap. + */ + if (!use_goto_tb || !is_same_page(&s->base, new_pc)) { + tcg_gen_andi_tl(cpu_eip, cpu_eip, mask); + use_goto_tb = false; + } + } + + if (use_goto_tb && + translator_use_goto_tb(&s->base, new_eip + s->cs_base)) { + /* jump to same page: we can use a direct jump */ + tcg_gen_goto_tb(tb_num); + if (!TARGET_TB_PCREL) { + tcg_gen_movi_tl(cpu_eip, new_eip); + } + tcg_gen_exit_tb(s->base.tb, tb_num); + s->base.is_jmp = DISAS_NORETURN; } else { - gen_jmp_im(s, eip); - gen_eob(s); + if (!TARGET_TB_PCREL) { + tcg_gen_movi_tl(cpu_eip, new_eip); + } + if (s->jmp_opt) { + gen_jr(s); /* jump to another page */ + } else { + gen_eob(s); /* exit to main loop */ + } } } -static void gen_jmp(DisasContext *s, target_ulong eip) +/* Jump to eip+diff, truncating to the current code size. */ +static void gen_jmp_rel_csize(DisasContext *s, int diff, int tb_num) { - gen_jmp_tb(s, eip, 0); + /* CODE64 ignores the OT argument, so we need not consider it. */ + gen_jmp_rel(s, CODE32(s) ? MO_32 : MO_16, diff, tb_num); } static inline void gen_ldq_env_A0(DisasContext *s, int offset) @@ -3226,8 +3326,7 @@ static const struct SSEOpHelper_table7 sse_op_table7[256] = { goto illegal_op; \ } while (0) -static void gen_sse(CPUX86State *env, DisasContext *s, int b, - target_ulong pc_start) +static void gen_sse(CPUX86State *env, DisasContext *s, int b) { int b1, op1_offset, op2_offset, is_xmm, val; int modrm, mod, rm, reg; @@ -3269,7 +3368,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b, } /* simple MMX/SSE operation */ if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); return; } if (s->flags & HF_EM_MASK) { @@ -4709,19 +4808,17 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b, /* convert one instruction. s->base.is_jmp is set if the translation must be stopped. Return the next pc value */ -static target_ulong disas_insn(DisasContext *s, CPUState *cpu) +static bool disas_insn(DisasContext *s, CPUState *cpu) { CPUX86State *env = cpu->env_ptr; int b, prefixes; int shift; MemOp ot, aflag, dflag; int modrm, reg, rm, mod, op, opreg, val; - target_ulong next_eip, tval; - target_ulong pc_start = s->base.pc_next; bool orig_cc_op_dirty = s->cc_op_dirty; CCOp orig_cc_op = s->cc_op; - s->pc_start = s->pc = pc_start; + s->pc = s->base.pc_next; s->override = -1; #ifdef TARGET_X86_64 s->rex_w = false; @@ -4737,15 +4834,16 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; case 1: gen_exception_gpf(s); - return s->pc; + return true; case 2: /* Restore state that may affect the next instruction. */ + s->pc = s->base.pc_next; s->cc_op_dirty = orig_cc_op_dirty; s->cc_op = orig_cc_op; s->base.num_insns--; tcg_remove_ops_after(s->prev_insn_end); s->base.is_jmp = DISAS_TOO_MANY; - return pc_start; + return false; default: g_assert_not_reached(); } @@ -5297,12 +5395,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (dflag == MO_16) { tcg_gen_ext16u_tl(s->T0, s->T0); } - next_eip = s->pc - s->cs_base; - tcg_gen_movi_tl(s->T1, next_eip); - gen_push_v(s, s->T1); - gen_op_jmp_v(s->T0); + gen_push_v(s, eip_next_tl(s)); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 3: /* lcall Ev */ if (mod == 3) { @@ -5315,24 +5411,24 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (PE(s) && !VM86(s)) { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); gen_helper_lcall_protected(cpu_env, s->tmp2_i32, s->T1, - tcg_const_i32(dflag - 1), - tcg_const_tl(s->pc - s->cs_base)); + tcg_constant_i32(dflag - 1), + eip_next_tl(s)); } else { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); - gen_helper_lcall_real(cpu_env, s->tmp2_i32, s->T1, - tcg_const_i32(dflag - 1), - tcg_const_i32(s->pc - s->cs_base)); + tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); + gen_helper_lcall_real(cpu_env, s->tmp2_i32, s->tmp3_i32, + tcg_constant_i32(dflag - 1), + eip_next_i32(s)); } - tcg_gen_ld_tl(s->tmp4, cpu_env, offsetof(CPUX86State, eip)); - gen_jr(s, s->tmp4); + s->base.is_jmp = DISAS_JUMP; break; case 4: /* jmp Ev */ if (dflag == MO_16) { tcg_gen_ext16u_tl(s->T0, s->T0); } - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 5: /* ljmp Ev */ if (mod == 3) { @@ -5345,13 +5441,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (PE(s) && !VM86(s)) { tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); gen_helper_ljmp_protected(cpu_env, s->tmp2_i32, s->T1, - tcg_const_tl(s->pc - s->cs_base)); + eip_next_tl(s)); } else { gen_op_movl_seg_T0_vm(s, R_CS); - gen_op_jmp_v(s->T1); + gen_op_jmp_v(s, s->T1); } - tcg_gen_ld_tl(s->tmp4, cpu_env, offsetof(CPUX86State, eip)); - gen_jr(s, s->tmp4); + s->base.is_jmp = DISAS_JUMP; break; case 6: /* push Ev */ gen_push_v(s, s->T0); @@ -5618,14 +5713,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_rdrand(s->T0, cpu_env); rm = (modrm & 7) | REX_B(s); gen_op_mov_reg_v(s, dflag, rm, s->T0); set_cc_op(s, CC_OP_EFLAGS); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; default: @@ -5716,26 +5809,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) ot = gen_pop_T0(s); gen_movl_seg_T0(s, reg); gen_pop_update(s, ot); - /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp. */ - if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); - if (reg == R_SS) { - s->flags &= ~HF_TF_MASK; - gen_eob_inhibit_irq(s, true); - } else { - gen_eob(s); - } - } break; case 0x1a1: /* pop fs */ case 0x1a9: /* pop gs */ ot = gen_pop_T0(s); gen_movl_seg_T0(s, (b >> 3) & 7); gen_pop_update(s, ot); - if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); - } break; /**************************/ @@ -5782,16 +5861,6 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0); gen_movl_seg_T0(s, reg); - /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp. */ - if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); - if (reg == R_SS) { - s->flags &= ~HF_TF_MASK; - gen_eob_inhibit_irq(s, true); - } else { - gen_eob(s); - } - } break; case 0x8c: /* mov Gv, seg */ modrm = x86_ldub_code(env, s); @@ -5981,10 +6050,6 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_movl_seg_T0(s, op); /* then put the data */ gen_op_mov_reg_v(s, ot, reg, s->T1); - if (s->base.is_jmp) { - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); - } break; /************************/ @@ -6079,7 +6144,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (s->flags & (HF_EM_MASK | HF_TS_MASK)) { /* if CR0.EM or CR0.TS are set, generate an FPU exception */ /* XXX: what to do if illegal op ? */ - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } modrm = x86_ldub_code(env, s); @@ -6620,7 +6685,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) offsetof(CPUX86State, segs[R_CS].selector)); tcg_gen_st16_i32(s->tmp2_i32, cpu_env, offsetof(CPUX86State, fpcs)); - tcg_gen_st_tl(tcg_constant_tl(pc_start - s->cs_base), + tcg_gen_st_tl(eip_cur_tl(s), cpu_env, offsetof(CPUX86State, fpip)); } } @@ -6632,7 +6697,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xa5: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_movs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_movs(s, ot); } else { gen_movs(s, ot); } @@ -6642,7 +6707,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xab: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_stos(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_stos(s, ot); } else { gen_stos(s, ot); } @@ -6651,7 +6716,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xad: ot = mo_b_d(b, dflag); if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_lods(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); + gen_repz_lods(s, ot); } else { gen_lods(s, ot); } @@ -6660,9 +6725,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xaf: ot = mo_b_d(b, dflag); if (prefixes & PREFIX_REPNZ) { - gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1); + gen_repz_scas(s, ot, 1); } else if (prefixes & PREFIX_REPZ) { - gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0); + gen_repz_scas(s, ot, 0); } else { gen_scas(s, ot); } @@ -6672,9 +6737,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xa7: ot = mo_b_d(b, dflag); if (prefixes & PREFIX_REPNZ) { - gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1); + gen_repz_cmps(s, ot, 1); } else if (prefixes & PREFIX_REPZ) { - gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0); + gen_repz_cmps(s, ot, 0); } else { gen_cmps(s, ot); } @@ -6690,15 +6755,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); - /* jump generated by gen_repz_ins */ + gen_repz_ins(s, ot); } else { gen_ins(s, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } } break; case 0x6e: /* outsS */ @@ -6711,15 +6773,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) { - gen_repz_outs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base); - /* jump generated by gen_repz_outs */ + gen_repz_outs(s, ot); } else { gen_outs(s, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } } break; @@ -6736,13 +6795,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_in_func(ot, s->T1, s->tmp2_i32); gen_op_mov_reg_v(s, ot, R_EAX, s->T1); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0xe6: case 0xe7: @@ -6754,14 +6811,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_op_mov_v_reg(s, ot, s->T1, R_EAX); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_out_func(ot, s->tmp2_i32, s->tmp3_i32); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0xec: case 0xed: @@ -6773,13 +6828,11 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_in_func(ot, s->T1, s->tmp2_i32); gen_op_mov_reg_v(s, ot, R_EAX, s->T1); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0xee: case 0xef: @@ -6791,14 +6844,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_op_mov_v_reg(s, ot, s->T1, R_EAX); tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); gen_helper_out_func(ot, s->tmp2_i32, s->tmp3_i32); gen_bpt_io(s, s->tmp2_i32, ot); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; /************************/ @@ -6808,24 +6859,24 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) ot = gen_pop_T0(s); gen_stack_update(s, val + (1 << ot)); /* Note that gen_pop_T0 uses a zero-extending load. */ - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 0xc3: /* ret */ ot = gen_pop_T0(s); gen_pop_update(s, ot); /* Note that gen_pop_T0 uses a zero-extending load. */ - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); gen_bnd_jmp(s); - gen_jr(s, s->T0); + s->base.is_jmp = DISAS_JUMP; break; case 0xca: /* lret im */ val = x86_ldsw_code(env, s); do_lret: if (PE(s) && !VM86(s)) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_lret_protected(cpu_env, tcg_const_i32(dflag - 1), tcg_const_i32(val)); } else { @@ -6834,7 +6885,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_op_ld_v(s, dflag, s->T0, s->A0); /* NOTE: keeping EIP updated is not a problem in case of exception */ - gen_op_jmp_v(s->T0); + gen_op_jmp_v(s, s->T0); /* pop selector */ gen_add_A0_im(s, 1 << dflag); gen_op_ld_v(s, dflag, s->T0, s->A0); @@ -6842,7 +6893,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) /* add stack offset */ gen_stack_update(s, val + (2 << dflag)); } - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; break; case 0xcb: /* lret */ val = 0; @@ -6856,30 +6907,20 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } gen_helper_iret_real(cpu_env, tcg_const_i32(dflag - 1)); } else { - gen_helper_iret_protected(cpu_env, tcg_const_i32(dflag - 1), - tcg_const_i32(s->pc - s->cs_base)); + gen_helper_iret_protected(cpu_env, tcg_constant_i32(dflag - 1), + eip_next_i32(s)); } set_cc_op(s, CC_OP_EFLAGS); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; break; case 0xe8: /* call im */ { - if (dflag != MO_16) { - tval = (int32_t)insn_get(env, s, MO_32); - } else { - tval = (int16_t)insn_get(env, s, MO_16); - } - next_eip = s->pc - s->cs_base; - tval += next_eip; - if (dflag == MO_16) { - tval &= 0xffff; - } else if (!CODE64(s)) { - tval &= 0xffffffff; - } - tcg_gen_movi_tl(s->T0, next_eip); - gen_push_v(s, s->T0); + int diff = (dflag != MO_16 + ? (int32_t)insn_get(env, s, MO_32) + : (int16_t)insn_get(env, s, MO_16)); + gen_push_v(s, eip_next_tl(s)); gen_bnd_jmp(s); - gen_jmp(s, tval); + gen_jmp_rel(s, dflag, diff, 0); } break; case 0x9a: /* lcall im */ @@ -6897,19 +6938,13 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } goto do_lcall; case 0xe9: /* jmp im */ - if (dflag != MO_16) { - tval = (int32_t)insn_get(env, s, MO_32); - } else { - tval = (int16_t)insn_get(env, s, MO_16); + { + int diff = (dflag != MO_16 + ? (int32_t)insn_get(env, s, MO_32) + : (int16_t)insn_get(env, s, MO_16)); + gen_bnd_jmp(s); + gen_jmp_rel(s, dflag, diff, 0); } - tval += s->pc - s->cs_base; - if (dflag == MO_16) { - tval &= 0xffff; - } else if (!CODE64(s)) { - tval &= 0xffffffff; - } - gen_bnd_jmp(s); - gen_jmp(s, tval); break; case 0xea: /* ljmp im */ { @@ -6926,30 +6961,26 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } goto do_ljmp; case 0xeb: /* jmp Jb */ - tval = (int8_t)insn_get(env, s, MO_8); - tval += s->pc - s->cs_base; - if (dflag == MO_16) { - tval &= 0xffff; + { + int diff = (int8_t)insn_get(env, s, MO_8); + gen_jmp_rel(s, dflag, diff, 0); } - gen_jmp(s, tval); break; case 0x70 ... 0x7f: /* jcc Jb */ - tval = (int8_t)insn_get(env, s, MO_8); - goto do_jcc; + { + int diff = (int8_t)insn_get(env, s, MO_8); + gen_bnd_jmp(s); + gen_jcc(s, b, diff); + } + break; case 0x180 ... 0x18f: /* jcc Jv */ - if (dflag != MO_16) { - tval = (int32_t)insn_get(env, s, MO_32); - } else { - tval = (int16_t)insn_get(env, s, MO_16); + { + int diff = (dflag != MO_16 + ? (int32_t)insn_get(env, s, MO_32) + : (int16_t)insn_get(env, s, MO_16)); + gen_bnd_jmp(s); + gen_jcc(s, b, diff); } - do_jcc: - next_eip = s->pc - s->cs_base; - tval += next_eip; - if (dflag == MO_16) { - tval &= 0xffff; - } - gen_bnd_jmp(s); - gen_jcc(s, b, tval, next_eip); break; case 0x190 ... 0x19f: /* setcc Gv */ @@ -7029,8 +7060,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_pop_update(s, ot); set_cc_op(s, CC_OP_EFLAGS); /* abort translation because TF/AC flag may change */ - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } break; case 0x9e: /* sahf */ @@ -7295,7 +7325,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; val = x86_ldub_code(env, s); if (val == 0) { - gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base); + gen_exception(s, EXCP00_DIVZ); } else { gen_helper_aam(cpu_env, tcg_const_i32(val)); set_cc_op(s, CC_OP_LOGICB); @@ -7321,34 +7351,34 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } if (prefixes & PREFIX_REPZ) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_pause(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_update_eip_cur(s); + gen_helper_pause(cpu_env, cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; } break; case 0x9b: /* fwait */ if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); } else { gen_helper_fwait(cpu_env); } break; case 0xcc: /* int3 */ - gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base); + gen_interrupt(s, EXCP03_INT3); break; case 0xcd: /* int N */ val = x86_ldub_code(env, s); if (check_vm86_iopl(s)) { - gen_interrupt(s, val, pc_start - s->cs_base, s->pc - s->cs_base); + gen_interrupt(s, val); } break; case 0xce: /* into */ if (CODE64(s)) goto illegal_op; gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_into(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_update_eip_cur(s); + gen_helper_into(cpu_env, cur_insn_len_i32(s)); break; #ifdef WANT_ICEBP case 0xf1: /* icebp (undocumented, exits to external debugger) */ @@ -7365,7 +7395,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (check_iopl(s)) { gen_helper_sti(cpu_env); /* interruptions are enabled only the first insn after sti */ - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_eob_inhibit_irq(s, true); } break; @@ -7409,75 +7439,62 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0xe2: /* loop */ case 0xe3: /* jecxz */ { - TCGLabel *l1, *l2, *l3; - - tval = (int8_t)insn_get(env, s, MO_8); - next_eip = s->pc - s->cs_base; - tval += next_eip; - if (dflag == MO_16) { - tval &= 0xffff; - } + TCGLabel *l1, *l2; + int diff = (int8_t)insn_get(env, s, MO_8); l1 = gen_new_label(); l2 = gen_new_label(); - l3 = gen_new_label(); gen_update_cc_op(s); b &= 3; switch(b) { case 0: /* loopnz */ case 1: /* loopz */ gen_op_add_reg_im(s, s->aflag, R_ECX, -1); - gen_op_jz_ecx(s, s->aflag, l3); + gen_op_jz_ecx(s, l2); gen_jcc1(s, (JCC_Z << 1) | (b ^ 1), l1); break; case 2: /* loop */ gen_op_add_reg_im(s, s->aflag, R_ECX, -1); - gen_op_jnz_ecx(s, s->aflag, l1); + gen_op_jnz_ecx(s, l1); break; default: case 3: /* jcxz */ - gen_op_jz_ecx(s, s->aflag, l1); + gen_op_jz_ecx(s, l1); break; } - gen_set_label(l3); - gen_jmp_im(s, next_eip); - tcg_gen_br(l2); + gen_set_label(l2); + gen_jmp_rel_csize(s, 0, 1); gen_set_label(l1); - gen_jmp_im(s, tval); - gen_set_label(l2); - gen_eob(s); + gen_jmp_rel(s, dflag, diff, 0); } break; case 0x130: /* wrmsr */ case 0x132: /* rdmsr */ if (check_cpl0(s)) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); if (b & 2) { gen_helper_rdmsr(cpu_env); } else { gen_helper_wrmsr(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } } break; case 0x131: /* rdtsc */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_rdtsc(cpu_env); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; case 0x133: /* rdpmc */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_rdpmc(cpu_env); s->base.is_jmp = DISAS_NORETURN; break; @@ -7489,7 +7506,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_exception_gpf(s); } else { gen_helper_sysenter(cpu_env); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; } break; case 0x135: /* sysexit */ @@ -7500,15 +7517,15 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_exception_gpf(s); } else { gen_helper_sysexit(cpu_env, tcg_const_i32(dflag - 1)); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; } break; #ifdef TARGET_X86_64 case 0x105: /* syscall */ /* XXX: is it usable in real mode ? */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_syscall(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_update_eip_cur(s); + gen_helper_syscall(cpu_env, cur_insn_len_i32(s)); /* TF handling for the syscall insn is different. The TF bit is checked after the syscall insn completes. This allows #DB to not be generated after one has entered CPL0 if TF is set in FMASK. */ @@ -7533,14 +7550,14 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) #endif case 0x1a2: /* cpuid */ gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_cpuid(cpu_env); break; case 0xf4: /* hlt */ if (check_cpl0(s)) { gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_hlt(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_update_eip_cur(s); + gen_helper_hlt(cpu_env, cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; } break; @@ -7636,7 +7653,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); tcg_gen_mov_tl(s->A0, cpu_regs[R_EAX]); gen_extu(s->aflag, s->A0); gen_add_A0_ds_seg(s); @@ -7648,8 +7665,8 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); - gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start)); + gen_update_eip_cur(s); + gen_helper_mwait(cpu_env, cur_insn_len_i32(s)); s->base.is_jmp = DISAS_NORETURN; break; @@ -7659,8 +7676,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_helper_clac(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xcb: /* stac */ @@ -7669,8 +7685,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_helper_stac(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(1): /* sidt */ @@ -7714,8 +7729,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_trunc_tl_i32(s->tmp2_i32, cpu_regs[R_ECX]); gen_helper_xsetbv(cpu_env, s->tmp2_i32, s->tmp1_i64); /* End TB because translation flags may change. */ - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xd8: /* VMRUN */ @@ -7726,9 +7740,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1), - tcg_const_i32(s->pc - pc_start)); + cur_insn_len_i32(s)); tcg_gen_exit_tb(NULL, 0); s->base.is_jmp = DISAS_NORETURN; break; @@ -7738,7 +7752,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmmcall(cpu_env); break; @@ -7750,7 +7764,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1)); break; @@ -7762,7 +7776,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1)); break; @@ -7776,8 +7790,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) } gen_update_cc_op(s); gen_helper_stgi(cpu_env); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xdd: /* CLGI */ @@ -7788,7 +7801,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) break; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); gen_helper_clgi(cpu_env); break; @@ -7815,8 +7828,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_ext32u_tl(s->A0, cpu_regs[R_EAX]); } gen_helper_flush_page(cpu_env, s->A0); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(2): /* lgdt */ @@ -7899,8 +7911,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) tcg_gen_andi_tl(s->T1, s->T1, ~0xe); tcg_gen_or_tl(s->T0, s->T0, s->T1); gen_helper_write_crN(cpu_env, tcg_constant_i32(0), s->T0); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(7): /* invlpg */ @@ -7910,8 +7921,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_INVLPG); gen_lea_modrm(env, s, modrm); gen_helper_flush_page(cpu_env, s->A0); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; case 0xf8: /* swapgs */ @@ -7934,14 +7944,12 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } gen_update_cc_op(s); - gen_jmp_im(s, pc_start - s->cs_base); + gen_update_eip_cur(s); if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } gen_helper_rdtscp(cpu_env); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } break; default: @@ -8305,20 +8313,17 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { gen_io_start(); + s->base.is_jmp = DISAS_TOO_MANY; } if (b & 2) { gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0 + reg); gen_op_mov_v_reg(s, ot, s->T0, rm); gen_helper_write_crN(cpu_env, tcg_constant_i32(reg), s->T0); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } else { gen_svm_check_intercept(s, SVM_EXIT_READ_CR0 + reg); gen_helper_read_crN(s->T0, cpu_env, tcg_constant_i32(reg)); gen_op_mov_reg_v(s, ot, rm, s->T0); - if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { - gen_jmp(s, s->pc - s->cs_base); - } } break; @@ -8345,8 +8350,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_op_mov_v_reg(s, ot, s->T0, rm); tcg_gen_movi_i32(s->tmp2_i32, reg); gen_helper_set_dr(cpu_env, s->tmp2_i32, s->T0); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } else { gen_svm_check_intercept(s, SVM_EXIT_READ_DR0 + reg); tcg_gen_movi_i32(s->tmp2_i32, reg); @@ -8360,8 +8364,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_svm_check_intercept(s, SVM_EXIT_WRITE_CR0); gen_helper_clts(cpu_env); /* abort block because static cpu state changed */ - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; } break; /* MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4 support */ @@ -8386,7 +8389,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_lea_modrm(env, s, modrm); @@ -8399,7 +8402,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_lea_modrm(env, s, modrm); @@ -8411,7 +8414,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_lea_modrm(env, s, modrm); @@ -8424,7 +8427,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) goto illegal_op; } if (s->flags & HF_TS_MASK) { - gen_exception(s, EXCP07_PREX, pc_start - s->cs_base); + gen_exception(s, EXCP07_PREX); break; } gen_helper_update_mxcsr(cpu_env); @@ -8457,9 +8460,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) gen_helper_xrstor(cpu_env, s->A0, s->tmp1_i64); /* XRSTOR is how MPX is enabled, which changes how we translate. Thus we need to end the TB. */ - gen_update_cc_op(s); - gen_jmp_im(s, s->pc - s->cs_base); - gen_eob(s); + s->base.is_jmp = DISAS_EOB_NEXT; break; CASE_MODRM_MEM_OP(6): /* xsaveopt / clwb */ @@ -8592,10 +8593,10 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) g_assert_not_reached(); #else gen_update_cc_op(s); - gen_jmp_im(s, s->pc - s->cs_base); + gen_update_eip_next(s); gen_helper_rsm(cpu_env); #endif /* CONFIG_USER_ONLY */ - gen_eob(s); + s->base.is_jmp = DISAS_EOB_ONLY; break; case 0x1b8: /* SSE4.2 popcnt */ if ((prefixes & (PREFIX_REPZ | PREFIX_LOCK | PREFIX_REPNZ)) != @@ -8633,18 +8634,18 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu) case 0x1c2: case 0x1c4 ... 0x1c6: case 0x1d0 ... 0x1fe: - gen_sse(env, s, b, pc_start); + gen_sse(env, s, b); break; default: goto unknown_op; } - return s->pc; + return true; illegal_op: gen_illegal_opcode(s); - return s->pc; + return true; unknown_op: gen_unknown_opcode(env, s); - return s->pc; + return true; } void tcg_x86_init(void) @@ -8676,6 +8677,13 @@ void tcg_x86_init(void) [R_EDI] = "edi", [R_EBP] = "ebp", [R_ESP] = "esp", +#endif + }; + static const char eip_name[] = { +#ifdef TARGET_X86_64 + "rip" +#else + "eip" #endif }; static const char seg_base_names[6][8] = { @@ -8702,6 +8710,7 @@ void tcg_x86_init(void) "cc_src"); cpu_cc_src2 = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_src2), "cc_src2"); + cpu_eip = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, eip), eip_name); for (i = 0; i < CPU_NB_REGS; ++i) { cpu_regs[i] = tcg_global_mem_new(cpu_env, @@ -8738,6 +8747,7 @@ static void i386_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cpu) int iopl = (flags >> IOPL_SHIFT) & 3; dc->cs_base = dc->base.tb->cs_base; + dc->pc_save = dc->base.pc_next; dc->flags = flags; #ifndef CONFIG_USER_ONLY dc->cpl = cpl; @@ -8801,42 +8811,48 @@ static void i386_tr_tb_start(DisasContextBase *db, CPUState *cpu) static void i386_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); + target_ulong pc_arg = dc->base.pc_next; dc->prev_insn_end = tcg_last_op(); - tcg_gen_insn_start(dc->base.pc_next, dc->cc_op); + if (TARGET_TB_PCREL) { + pc_arg -= dc->cs_base; + pc_arg &= ~TARGET_PAGE_MASK; + } + tcg_gen_insn_start(pc_arg, dc->cc_op); } static void i386_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); - target_ulong pc_next; #ifdef TARGET_VSYSCALL_PAGE /* * Detect entry into the vsyscall page and invoke the syscall. */ if ((dc->base.pc_next & TARGET_PAGE_MASK) == TARGET_VSYSCALL_PAGE) { - gen_exception(dc, EXCP_VSYSCALL, dc->base.pc_next); + gen_exception(dc, EXCP_VSYSCALL); dc->base.pc_next = dc->pc + 1; return; } #endif - pc_next = disas_insn(dc, cpu); - dc->base.pc_next = pc_next; + if (disas_insn(dc, cpu)) { + target_ulong pc_next = dc->pc; + dc->base.pc_next = pc_next; - if (dc->base.is_jmp == DISAS_NEXT) { - if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { - /* - * If single step mode, we generate only one instruction and - * generate an exception. - * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear - * the flag and abort the translation to give the irqs a - * chance to happen. - */ - dc->base.is_jmp = DISAS_TOO_MANY; - } else if (!is_same_page(&dc->base, pc_next)) { - dc->base.is_jmp = DISAS_TOO_MANY; + if (dc->base.is_jmp == DISAS_NEXT) { + if (dc->flags & (HF_TF_MASK | HF_INHIBIT_IRQ_MASK)) { + /* + * If single step mode, we generate only one instruction and + * generate an exception. + * If irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear + * the flag and abort the translation to give the irqs a + * chance to happen. + */ + dc->base.is_jmp = DISAS_EOB_NEXT; + } else if (!is_same_page(&dc->base, pc_next)) { + dc->base.is_jmp = DISAS_TOO_MANY; + } } } } @@ -8845,9 +8861,30 @@ static void i386_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) { DisasContext *dc = container_of(dcbase, DisasContext, base); - if (dc->base.is_jmp == DISAS_TOO_MANY) { - gen_jmp_im(dc, dc->base.pc_next - dc->cs_base); + switch (dc->base.is_jmp) { + case DISAS_NORETURN: + break; + case DISAS_TOO_MANY: + gen_update_cc_op(dc); + gen_jmp_rel_csize(dc, 0, 0); + break; + case DISAS_EOB_NEXT: + gen_update_cc_op(dc); + gen_update_eip_cur(dc); + /* fall through */ + case DISAS_EOB_ONLY: gen_eob(dc); + break; + case DISAS_EOB_INHIBIT_IRQ: + gen_update_cc_op(dc); + gen_update_eip_cur(dc); + gen_eob_inhibit_irq(dc, true); + break; + case DISAS_JUMP: + gen_jr(dc); + break; + default: + g_assert_not_reached(); } } @@ -8882,7 +8919,12 @@ void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, target_ulong *data) { int cc_op = data[1]; - env->eip = data[0] - tb->cs_base; + + if (TARGET_TB_PCREL) { + env->eip = (env->eip & TARGET_PAGE_MASK) | data[0]; + } else { + env->eip = data[0] - tb->cs_base; + } if (cc_op != CC_OP_DYNAMIC) { env->cc_op = cc_op; } diff --git a/target/mips/kvm.c b/target/mips/kvm.c index caf70decd2..bcb8e06b2c 100644 --- a/target/mips/kvm.c +++ b/target/mips/kvm.c @@ -1294,3 +1294,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/ppc/kvm.c b/target/ppc/kvm.c index 466d0d2f4c..7c25348b7b 100644 --- a/target/ppc/kvm.c +++ b/target/ppc/kvm.c @@ -2966,3 +2966,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/riscv/kvm.c b/target/riscv/kvm.c index 70b4cff06f..30f21453d6 100644 --- a/target/riscv/kvm.c +++ b/target/riscv/kvm.c @@ -532,3 +532,7 @@ bool kvm_arch_cpu_check_are_resettable(void) { return true; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +} diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c index 6a8dbadf7e..508c24cfec 100644 --- a/target/s390x/kvm/kvm.c +++ b/target/s390x/kvm/kvm.c @@ -2581,3 +2581,7 @@ int kvm_s390_get_zpci_op(void) { return cap_zpci_op; } + +void kvm_arch_accel_class_init(ObjectClass *oc) +{ +}