From b264f57fde0c686c5c1dfdd0c21992c49196bb87 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Sun, 27 Oct 2019 16:19:38 +0100 Subject: [PATCH 1/3] x86/hyperv: Micro-optimize send_ipi_one() When sending an IPI to a single CPU there is no need to deal with cpumasks. With 2 CPU guest on WS2019 a minor (like 3%, 8043 -> 7761 CPU cycles) improvement with smp_call_function_single() loop benchmark can be seeb. The optimization, however, is tiny and straitforward. Also, send_ipi_one() is important for PV spinlock kick. Switching to the regular APIC IPI send for CPU > 64 case does not make sense as it is twice as expesive (12650 CPU cycles for __send_ipi_mask_ex() call, 26000 for orig_apic.send_IPI(cpu, vector)). Signed-off-by: Vitaly Kuznetsov Signed-off-by: Thomas Gleixner Reviewed-by: Michael Kelley Reviewed-by: Roman Kagan Link: https://lkml.kernel.org/r/20191027151938.7296-1-vkuznets@redhat.com --- arch/x86/hyperv/hv_apic.c | 16 +++++++++++++--- arch/x86/include/asm/trace/hyperv.h | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c index 5c056b8aebef..86c8674e92f4 100644 --- a/arch/x86/hyperv/hv_apic.c +++ b/arch/x86/hyperv/hv_apic.c @@ -194,10 +194,20 @@ do_ex_hypercall: static bool __send_ipi_one(int cpu, int vector) { - struct cpumask mask = CPU_MASK_NONE; + int vp = hv_cpu_number_to_vp_number(cpu); - cpumask_set_cpu(cpu, &mask); - return __send_ipi_mask(&mask, vector); + trace_hyperv_send_ipi_one(cpu, vector); + + if (!hv_hypercall_pg || (vp == VP_INVAL)) + return false; + + if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) + return false; + + if (vp >= 64) + return __send_ipi_mask_ex(cpumask_of(cpu), vector); + + return !hv_do_fast_hypercall16(HVCALL_SEND_IPI, vector, BIT_ULL(vp)); } static void hv_send_ipi(int cpu, int vector) diff --git a/arch/x86/include/asm/trace/hyperv.h b/arch/x86/include/asm/trace/hyperv.h index ace464f09681..4d705cb4d63b 100644 --- a/arch/x86/include/asm/trace/hyperv.h +++ b/arch/x86/include/asm/trace/hyperv.h @@ -71,6 +71,21 @@ TRACE_EVENT(hyperv_send_ipi_mask, __entry->ncpus, __entry->vector) ); +TRACE_EVENT(hyperv_send_ipi_one, + TP_PROTO(int cpu, + int vector), + TP_ARGS(cpu, vector), + TP_STRUCT__entry( + __field(int, cpu) + __field(int, vector) + ), + TP_fast_assign(__entry->cpu = cpu; + __entry->vector = vector; + ), + TP_printk("cpu %d vector %x", + __entry->cpu, __entry->vector) + ); + #endif /* CONFIG_HYPERV */ #undef TRACE_INCLUDE_PATH From dce7cd62754b5d4a6e401b8b0769ec94cf971041 Mon Sep 17 00:00:00 2001 From: Andrea Parri Date: Thu, 3 Oct 2019 17:52:00 +0200 Subject: [PATCH 2/3] x86/hyperv: Allow guests to enable InvariantTSC If the hardware supports TSC scaling, Hyper-V will set bit 15 of the HV_PARTITION_PRIVILEGE_MASK in guest VMs with a compatible Hyper-V configuration version. Bit 15 corresponds to the AccessTscInvariantControls privilege. If this privilege bit is set, guests can access the HvSyntheticInvariantTscControl MSR: guests can set bit 0 of this synthetic MSR to enable the InvariantTSC feature. After setting the synthetic MSR, CPUID will enumerate support for InvariantTSC. Signed-off-by: Andrea Parri Signed-off-by: Thomas Gleixner Reviewed-by: Michael Kelley Reviewed-by: Vitaly Kuznetsov Link: https://lkml.kernel.org/r/20191003155200.22022-1-parri.andrea@gmail.com --- arch/x86/include/asm/hyperv-tlfs.h | 5 +++++ arch/x86/kernel/cpu/mshyperv.c | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 7a2705694f5b..887b1d6a15ab 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -86,6 +86,8 @@ #define HV_X64_ACCESS_FREQUENCY_MSRS BIT(11) /* AccessReenlightenmentControls privilege */ #define HV_X64_ACCESS_REENLIGHTENMENT BIT(13) +/* AccessTscInvariantControls privilege */ +#define HV_X64_ACCESS_TSC_INVARIANT BIT(15) /* * Feature identification: indicates which flags were specified at partition @@ -270,6 +272,9 @@ #define HV_X64_MSR_TSC_EMULATION_CONTROL 0x40000107 #define HV_X64_MSR_TSC_EMULATION_STATUS 0x40000108 +/* TSC invariant control */ +#define HV_X64_MSR_TSC_INVARIANT_CONTROL 0x40000118 + /* * Declare the MSR used to setup pages used to communicate with the hypervisor. */ diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 062f77279ce3..6f7c82263993 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -285,7 +285,12 @@ static void __init ms_hyperv_init_platform(void) machine_ops.shutdown = hv_machine_shutdown; machine_ops.crash_shutdown = hv_machine_crash_shutdown; #endif - mark_tsc_unstable("running on Hyper-V"); + if (ms_hyperv.features & HV_X64_ACCESS_TSC_INVARIANT) { + wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, 0x1); + setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); + } else { + mark_tsc_unstable("running on Hyper-V"); + } /* * Generation 2 instances don't support reading the NMI status from From 4df4cb9e99f83b70d54bc0e25081ac23cceafcbc Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Wed, 13 Nov 2019 01:11:49 +0000 Subject: [PATCH 3/3] x86/hyperv: Initialize clockevents earlier in CPU onlining Hyper-V has historically initialized stimer-based clockevents late in the process of onlining a CPU because clockevents depend on stimer interrupts. In the original Hyper-V design, stimer interrupts generate a VMbus message, so the VMbus machinery must be running first, and VMbus can't be initialized until relatively late. On x86/64, LAPIC timer based clockevents are used during early initialization before VMbus and stimer-based clockevents are ready, and again during CPU offlining after the stimer clockevents have been shut down. Unfortunately, this design creates problems when offlining CPUs for hibernation or other purposes. stimer-based clockevents are shut down relatively early in the offlining process, so clockevents_unbind_device() must be used to fallback to the LAPIC-based clockevents for the remainder of the offlining process. Furthermore, the late initialization and early shutdown of stimer-based clockevents doesn't work well on ARM64 since there is no other timer like the LAPIC to fallback to. So CPU onlining and offlining doesn't work properly. Fix this by recognizing that stimer Direct Mode is the normal path for newer versions of Hyper-V on x86/64, and the only path on other architectures. With stimer Direct Mode, stimer interrupts don't require any VMbus machinery. stimer clockevents can be initialized and shut down consistent with how it is done for other clockevent devices. While the old VMbus-based stimer interrupts must still be supported for backward compatibility on x86, that mode of operation can be treated as legacy. So add a new Hyper-V stimer entry in the CPU hotplug state list, and use that new state when in Direct Mode. Update the Hyper-V clocksource driver to allocate and initialize stimer clockevents earlier during boot. Update Hyper-V initialization and the VMbus driver to use this new design. As a result, the LAPIC timer is no longer used during boot or CPU onlining/offlining and clockevents_unbind_device() is not called. But retain the old design as a legacy implementation for older versions of Hyper-V that don't support Direct Mode. Signed-off-by: Michael Kelley Signed-off-by: Thomas Gleixner Tested-by: Dexuan Cui Reviewed-by: Dexuan Cui Link: https://lkml.kernel.org/r/1573607467-9456-1-git-send-email-mikelley@microsoft.com --- arch/x86/hyperv/hv_init.c | 6 ++ drivers/clocksource/hyperv_timer.c | 156 +++++++++++++++++++++++------ drivers/hv/hv.c | 4 +- drivers/hv/vmbus_drv.c | 30 +++--- include/clocksource/hyperv_timer.h | 7 +- include/linux/cpuhotplug.h | 1 + 6 files changed, 152 insertions(+), 52 deletions(-) diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 2db3972c0e0f..50ff030d9224 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -311,6 +311,12 @@ void __init hyperv_init(void) hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + /* + * Ignore any errors in setting up stimer clockevents + * as we can run with the LAPIC timer as a fallback. + */ + (void)hv_stimer_alloc(); + hv_apic_init(); x86_init.pci.arch_init = hv_pci_init; diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index 2317d4e3daaf..287d8d58c21a 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,15 @@ static u64 hv_sched_clock_offset __ro_after_init; * mechanism is used when running on older versions of Hyper-V * that don't support Direct Mode. While Hyper-V provides * four stimer's per CPU, Linux uses only stimer0. + * + * Because Direct Mode does not require processing a VMbus + * message, stimer interrupts can be enabled earlier in the + * process of booting a CPU, and consistent with when timer + * interrupts are enabled for other clocksource drivers. + * However, for legacy versions of Hyper-V when Direct Mode + * is not enabled, setting up stimer interrupts must be + * delayed until VMbus is initialized and can process the + * interrupt message. */ static bool direct_mode_enabled; @@ -102,17 +112,12 @@ static int hv_ce_set_oneshot(struct clock_event_device *evt) /* * hv_stimer_init - Per-cpu initialization of the clockevent */ -void hv_stimer_init(unsigned int cpu) +static int hv_stimer_init(unsigned int cpu) { struct clock_event_device *ce; - /* - * Synthetic timers are always available except on old versions of - * Hyper-V on x86. In that case, just return as Linux will use a - * clocksource based on emulated PIT or LAPIC timer hardware. - */ - if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) - return; + if (!hv_clock_event) + return 0; ce = per_cpu_ptr(hv_clock_event, cpu); ce->name = "Hyper-V clockevent"; @@ -127,28 +132,55 @@ void hv_stimer_init(unsigned int cpu) HV_CLOCK_HZ, HV_MIN_DELTA_TICKS, HV_MAX_MAX_DELTA_TICKS); + return 0; } -EXPORT_SYMBOL_GPL(hv_stimer_init); /* * hv_stimer_cleanup - Per-cpu cleanup of the clockevent */ -void hv_stimer_cleanup(unsigned int cpu) +int hv_stimer_cleanup(unsigned int cpu) { struct clock_event_device *ce; - /* Turn off clockevent device */ - if (ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE) { - ce = per_cpu_ptr(hv_clock_event, cpu); + if (!hv_clock_event) + return 0; + + /* + * In the legacy case where Direct Mode is not enabled + * (which can only be on x86/64), stimer cleanup happens + * relatively early in the CPU offlining process. We + * must unbind the stimer-based clockevent device so + * that the LAPIC timer can take over until clockevents + * are no longer needed in the offlining process. Note + * that clockevents_unbind_device() eventually calls + * hv_ce_shutdown(). + * + * The unbind should not be done when Direct Mode is + * enabled because we may be on an architecture where + * there are no other clockevent devices to fallback to. + */ + ce = per_cpu_ptr(hv_clock_event, cpu); + if (direct_mode_enabled) hv_ce_shutdown(ce); - } + else + clockevents_unbind_device(ce, cpu); + + return 0; } EXPORT_SYMBOL_GPL(hv_stimer_cleanup); /* hv_stimer_alloc - Global initialization of the clockevent and stimer0 */ -int hv_stimer_alloc(int sint) +int hv_stimer_alloc(void) { - int ret; + int ret = 0; + + /* + * Synthetic timers are always available except on old versions of + * Hyper-V on x86. In that case, return as error as Linux will use a + * clockevent based on emulated LAPIC timer hardware. + */ + if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) + return -EINVAL; hv_clock_event = alloc_percpu(struct clock_event_device); if (!hv_clock_event) @@ -159,22 +191,78 @@ int hv_stimer_alloc(int sint) if (direct_mode_enabled) { ret = hv_setup_stimer0_irq(&stimer0_irq, &stimer0_vector, hv_stimer0_isr); - if (ret) { - free_percpu(hv_clock_event); - hv_clock_event = NULL; - return ret; - } - } + if (ret) + goto free_percpu; - stimer0_message_sint = sint; - return 0; + /* + * Since we are in Direct Mode, stimer initialization + * can be done now with a CPUHP value in the same range + * as other clockevent devices. + */ + ret = cpuhp_setup_state(CPUHP_AP_HYPERV_TIMER_STARTING, + "clockevents/hyperv/stimer:starting", + hv_stimer_init, hv_stimer_cleanup); + if (ret < 0) + goto free_stimer0_irq; + } + return ret; + +free_stimer0_irq: + hv_remove_stimer0_irq(stimer0_irq); + stimer0_irq = 0; +free_percpu: + free_percpu(hv_clock_event); + hv_clock_event = NULL; + return ret; } EXPORT_SYMBOL_GPL(hv_stimer_alloc); +/* + * hv_stimer_legacy_init -- Called from the VMbus driver to handle + * the case when Direct Mode is not enabled, and the stimer + * must be initialized late in the CPU onlining process. + * + */ +void hv_stimer_legacy_init(unsigned int cpu, int sint) +{ + if (direct_mode_enabled) + return; + + /* + * This function gets called by each vCPU, so setting the + * global stimer_message_sint value each time is conceptually + * not ideal, but the value passed in is always the same and + * it avoids introducing yet another interface into this + * clocksource driver just to set the sint in the legacy case. + */ + stimer0_message_sint = sint; + (void)hv_stimer_init(cpu); +} +EXPORT_SYMBOL_GPL(hv_stimer_legacy_init); + +/* + * hv_stimer_legacy_cleanup -- Called from the VMbus driver to + * handle the case when Direct Mode is not enabled, and the + * stimer must be cleaned up early in the CPU offlining + * process. + */ +void hv_stimer_legacy_cleanup(unsigned int cpu) +{ + if (direct_mode_enabled) + return; + (void)hv_stimer_cleanup(cpu); +} +EXPORT_SYMBOL_GPL(hv_stimer_legacy_cleanup); + + /* hv_stimer_free - Free global resources allocated by hv_stimer_alloc() */ void hv_stimer_free(void) { - if (direct_mode_enabled && (stimer0_irq != 0)) { + if (!hv_clock_event) + return; + + if (direct_mode_enabled) { + cpuhp_remove_state(CPUHP_AP_HYPERV_TIMER_STARTING); hv_remove_stimer0_irq(stimer0_irq); stimer0_irq = 0; } @@ -190,14 +278,20 @@ EXPORT_SYMBOL_GPL(hv_stimer_free); void hv_stimer_global_cleanup(void) { int cpu; - struct clock_event_device *ce; - if (ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE) { - for_each_present_cpu(cpu) { - ce = per_cpu_ptr(hv_clock_event, cpu); - clockevents_unbind_device(ce, cpu); - } + /* + * hv_stime_legacy_cleanup() will stop the stimer if Direct + * Mode is not enabled, and fallback to the LAPIC timer. + */ + for_each_present_cpu(cpu) { + hv_stimer_legacy_cleanup(cpu); } + + /* + * If Direct Mode is enabled, the cpuhp teardown callback + * (hv_stimer_cleanup) will be run on all CPUs to stop the + * stimers. + */ hv_stimer_free(); } EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index fcc52797c169..6098e0cbdb4b 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -202,7 +202,7 @@ int hv_synic_init(unsigned int cpu) { hv_synic_enable_regs(cpu); - hv_stimer_init(cpu); + hv_stimer_legacy_init(cpu, VMBUS_MESSAGE_SINT); return 0; } @@ -277,7 +277,7 @@ int hv_synic_cleanup(unsigned int cpu) if (channel_found && vmbus_connection.conn_state == CONNECTED) return -EBUSY; - hv_stimer_cleanup(cpu); + hv_stimer_legacy_cleanup(cpu); hv_synic_disable_regs(cpu); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 53a60c81e220..8c06b3361c27 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1340,10 +1340,6 @@ static int vmbus_bus_init(void) if (ret) goto err_alloc; - ret = hv_stimer_alloc(VMBUS_MESSAGE_SINT); - if (ret < 0) - goto err_alloc; - /* * Initialize the per-cpu interrupt state and stimer state. * Then connect to the host. @@ -1400,9 +1396,8 @@ static int vmbus_bus_init(void) err_connect: cpuhp_remove_state(hyperv_cpuhp_online); err_cpuhp: - hv_stimer_free(); -err_alloc: hv_synic_free(); +err_alloc: hv_remove_vmbus_irq(); bus_unregister(&hv_bus); @@ -2315,20 +2310,23 @@ static void hv_crash_handler(struct pt_regs *regs) static int hv_synic_suspend(void) { /* - * When we reach here, all the non-boot CPUs have been offlined, and - * the stimers on them have been unbound in hv_synic_cleanup() -> + * When we reach here, all the non-boot CPUs have been offlined. + * If we're in a legacy configuration where stimer Direct Mode is + * not enabled, the stimers on the non-boot CPUs have been unbound + * in hv_synic_cleanup() -> hv_stimer_legacy_cleanup() -> * hv_stimer_cleanup() -> clockevents_unbind_device(). * - * hv_synic_suspend() only runs on CPU0 with interrupts disabled. Here - * we do not unbind the stimer on CPU0 because: 1) it's unnecessary - * because the interrupts remain disabled between syscore_suspend() - * and syscore_resume(): see create_image() and resume_target_kernel(); + * hv_synic_suspend() only runs on CPU0 with interrupts disabled. + * Here we do not call hv_stimer_legacy_cleanup() on CPU0 because: + * 1) it's unnecessary as interrupts remain disabled between + * syscore_suspend() and syscore_resume(): see create_image() and + * resume_target_kernel() * 2) the stimer on CPU0 is automatically disabled later by * syscore_suspend() -> timekeeping_suspend() -> tick_suspend() -> ... - * -> clockevents_shutdown() -> ... -> hv_ce_shutdown(); 3) a warning - * would be triggered if we call clockevents_unbind_device(), which - * may sleep, in an interrupts-disabled context. So, we intentionally - * don't call hv_stimer_cleanup(0) here. + * -> clockevents_shutdown() -> ... -> hv_ce_shutdown() + * 3) a warning would be triggered if we call + * clockevents_unbind_device(), which may sleep, in an + * interrupts-disabled context. */ hv_synic_disable_regs(0); diff --git a/include/clocksource/hyperv_timer.h b/include/clocksource/hyperv_timer.h index 422f5e5237be..553e539469f0 100644 --- a/include/clocksource/hyperv_timer.h +++ b/include/clocksource/hyperv_timer.h @@ -21,10 +21,11 @@ #define HV_MIN_DELTA_TICKS 1 /* Routines called by the VMbus driver */ -extern int hv_stimer_alloc(int sint); +extern int hv_stimer_alloc(void); extern void hv_stimer_free(void); -extern void hv_stimer_init(unsigned int cpu); -extern void hv_stimer_cleanup(unsigned int cpu); +extern int hv_stimer_cleanup(unsigned int cpu); +extern void hv_stimer_legacy_init(unsigned int cpu, int sint); +extern void hv_stimer_legacy_cleanup(unsigned int cpu); extern void hv_stimer_global_cleanup(void); extern void hv_stimer0_isr(void); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 068793a619ca..4dcaea13b1f0 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -129,6 +129,7 @@ enum cpuhp_state { CPUHP_AP_ARC_TIMER_STARTING, CPUHP_AP_RISCV_TIMER_STARTING, CPUHP_AP_CSKY_TIMER_STARTING, + CPUHP_AP_HYPERV_TIMER_STARTING, CPUHP_AP_KVM_STARTING, CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING, CPUHP_AP_KVM_ARM_VGIC_STARTING,