2013-01-21 00:28:06 +01:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2012 - Virtual Open Systems and Columbia University
|
|
|
|
* Author: Christoffer Dall <c.dall@virtualopensystems.com>
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License, version 2, as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
|
|
|
*/
|
2013-01-21 00:28:06 +01:00
|
|
|
|
|
|
|
#include <linux/mman.h>
|
|
|
|
#include <linux/kvm_host.h>
|
|
|
|
#include <linux/io.h>
|
2012-11-01 17:14:45 +01:00
|
|
|
#include <linux/hugetlb.h>
|
2013-01-21 00:43:58 +01:00
|
|
|
#include <trace/events/kvm.h>
|
2013-01-21 00:28:06 +01:00
|
|
|
#include <asm/pgalloc.h>
|
2013-01-21 00:28:12 +01:00
|
|
|
#include <asm/cacheflush.h>
|
2013-01-21 00:28:06 +01:00
|
|
|
#include <asm/kvm_arm.h>
|
|
|
|
#include <asm/kvm_mmu.h>
|
2013-01-21 00:43:58 +01:00
|
|
|
#include <asm/kvm_mmio.h>
|
2013-01-21 00:28:07 +01:00
|
|
|
#include <asm/kvm_asm.h>
|
2013-01-21 00:28:12 +01:00
|
|
|
#include <asm/kvm_emulate.h>
|
2015-01-29 12:59:54 +01:00
|
|
|
#include <asm/virt.h>
|
2013-01-21 00:28:07 +01:00
|
|
|
|
|
|
|
#include "trace.h"
|
2013-01-21 00:28:06 +01:00
|
|
|
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
static pgd_t *boot_hyp_pgd;
|
2013-04-12 20:12:03 +02:00
|
|
|
static pgd_t *hyp_pgd;
|
2015-03-19 17:42:28 +01:00
|
|
|
static pgd_t *merged_hyp_pgd;
|
2013-01-21 00:28:06 +01:00
|
|
|
static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
|
|
|
|
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
static unsigned long hyp_idmap_start;
|
|
|
|
static unsigned long hyp_idmap_end;
|
|
|
|
static phys_addr_t hyp_idmap_vector;
|
|
|
|
|
2016-03-22 18:01:21 +01:00
|
|
|
#define S2_PGD_SIZE (PTRS_PER_S2_PGD * sizeof(pgd_t))
|
2014-10-10 12:14:28 +02:00
|
|
|
#define hyp_pgd_order get_order(PTRS_PER_PGD * sizeof(pgd_t))
|
2014-03-28 15:25:19 +01:00
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
#define KVM_S2PTE_FLAG_IS_IOMAP (1UL << 0)
|
|
|
|
#define KVM_S2_FLAG_LOGGING_ACTIVE (1UL << 1)
|
|
|
|
|
|
|
|
static bool memslot_is_logging(struct kvm_memory_slot *memslot)
|
|
|
|
{
|
|
|
|
return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
|
2015-01-16 00:59:01 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8
|
|
|
|
* @kvm: pointer to kvm structure.
|
|
|
|
*
|
|
|
|
* Interface to HYP function to flush all VM TLB entries
|
|
|
|
*/
|
|
|
|
void kvm_flush_remote_tlbs(struct kvm *kvm)
|
|
|
|
{
|
|
|
|
kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
|
2015-01-16 00:58:58 +01:00
|
|
|
}
|
2012-11-01 17:14:45 +01:00
|
|
|
|
2013-01-28 16:27:00 +01:00
|
|
|
static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
|
2013-01-21 00:28:07 +01:00
|
|
|
{
|
2016-03-22 18:14:25 +01:00
|
|
|
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
|
2013-01-21 00:28:07 +01:00
|
|
|
}
|
|
|
|
|
2014-12-19 17:48:06 +01:00
|
|
|
/*
|
|
|
|
* D-Cache management functions. They take the page table entries by
|
|
|
|
* value, as they are flushing the cache using the kernel mapping (or
|
|
|
|
* kmap on 32bit).
|
|
|
|
*/
|
|
|
|
static void kvm_flush_dcache_pte(pte_t pte)
|
|
|
|
{
|
|
|
|
__kvm_flush_dcache_pte(pte);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void kvm_flush_dcache_pmd(pmd_t pmd)
|
|
|
|
{
|
|
|
|
__kvm_flush_dcache_pmd(pmd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void kvm_flush_dcache_pud(pud_t pud)
|
|
|
|
{
|
|
|
|
__kvm_flush_dcache_pud(pud);
|
|
|
|
}
|
|
|
|
|
ARM/arm64: KVM: test properly for a PTE's uncachedness
The open coded tests for checking whether a PTE maps a page as
uncached use a flawed '(pte_val(xxx) & CONST) != CONST' pattern,
which is not guaranteed to work since the type of a mapping is
not a set of mutually exclusive bits
For HYP mappings, the type is an index into the MAIR table (i.e, the
index itself does not contain any information whatsoever about the
type of the mapping), and for stage-2 mappings it is a bit field where
normal memory and device types are defined as follows:
#define MT_S2_NORMAL 0xf
#define MT_S2_DEVICE_nGnRE 0x1
I.e., masking *and* comparing with the latter matches on the former,
and we have been getting lucky merely because the S2 device mappings
also have the PTE_UXN bit set, or we would misidentify memory mappings
as device mappings.
Since the unmap_range() code path (which contains one instance of the
flawed test) is used both for HYP mappings and stage-2 mappings, and
considering the difference between the two, it is non-trivial to fix
this by rewriting the tests in place, as it would involve passing
down the type of mapping through all the functions.
However, since HYP mappings and stage-2 mappings both deal with host
physical addresses, we can simply check whether the mapping is backed
by memory that is managed by the host kernel, and only perform the
D-cache maintenance if this is the case.
Cc: stable@vger.kernel.org
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: Pavel Fedin <p.fedin@samsung.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2015-11-10 15:11:20 +01:00
|
|
|
static bool kvm_is_device_pfn(unsigned long pfn)
|
|
|
|
{
|
|
|
|
return !pfn_valid(pfn);
|
|
|
|
}
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
/**
|
|
|
|
* stage2_dissolve_pmd() - clear and flush huge PMD entry
|
|
|
|
* @kvm: pointer to kvm structure.
|
|
|
|
* @addr: IPA
|
|
|
|
* @pmd: pmd pointer for IPA
|
|
|
|
*
|
|
|
|
* Function clears a PMD entry, flushes addr 1st and 2nd stage TLBs. Marks all
|
|
|
|
* pages in the range dirty.
|
|
|
|
*/
|
|
|
|
static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)
|
|
|
|
{
|
2016-03-01 13:00:39 +01:00
|
|
|
if (!pmd_thp_or_huge(*pmd))
|
2015-01-16 00:58:58 +01:00
|
|
|
return;
|
|
|
|
|
|
|
|
pmd_clear(pmd);
|
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
|
|
|
put_page(virt_to_page(pmd));
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:07 +01:00
|
|
|
static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
|
|
|
|
int min, int max)
|
|
|
|
{
|
|
|
|
void *page;
|
|
|
|
|
|
|
|
BUG_ON(max > KVM_NR_MEM_OBJS);
|
|
|
|
if (cache->nobjs >= min)
|
|
|
|
return 0;
|
|
|
|
while (cache->nobjs < max) {
|
|
|
|
page = (void *)__get_free_page(PGALLOC_GFP);
|
|
|
|
if (!page)
|
|
|
|
return -ENOMEM;
|
|
|
|
cache->objects[cache->nobjs++] = page;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
|
|
|
|
{
|
|
|
|
while (mc->nobjs)
|
|
|
|
free_page((unsigned long)mc->objects[--mc->nobjs]);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
|
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
|
|
|
|
BUG_ON(!mc || !mc->nobjs);
|
|
|
|
p = mc->objects[--mc->nobjs];
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
|
arm64: KVM: fix 2-level page tables unmapping
When using 64kB pages, we only have two levels of page tables,
meaning that PGD, PUD and PMD are fused. In this case, trying
to refcount PUDs and PMDs independently is a a complete disaster,
as they are the same.
We manage to get it right for the allocation (stage2_set_pte uses
{pmd,pud}_none), but the unmapping path clears both pud and pmd
refcounts, which fails spectacularly with 2-level page tables.
The fix is to avoid calling clear_pud_entry when both the pmd and
pud pages are empty. For this, and instead of introducing another
pud_empty function, consolidate both pte_empty and pmd_empty into
page_empty (the code is actually identical) and use that to also
test the validity of the pud.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2013-08-06 14:05:48 +02:00
|
|
|
{
|
2016-03-23 13:08:02 +01:00
|
|
|
pud_t *pud_table __maybe_unused = stage2_pud_offset(pgd, 0UL);
|
|
|
|
stage2_pgd_clear(pgd);
|
2014-05-09 23:31:31 +02:00
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2016-03-23 13:08:02 +01:00
|
|
|
stage2_pud_free(pud_table);
|
2014-05-09 23:31:31 +02:00
|
|
|
put_page(virt_to_page(pgd));
|
arm64: KVM: fix 2-level page tables unmapping
When using 64kB pages, we only have two levels of page tables,
meaning that PGD, PUD and PMD are fused. In this case, trying
to refcount PUDs and PMDs independently is a a complete disaster,
as they are the same.
We manage to get it right for the allocation (stage2_set_pte uses
{pmd,pud}_none), but the unmapping path clears both pud and pmd
refcounts, which fails spectacularly with 2-level page tables.
The fix is to avoid calling clear_pud_entry when both the pmd and
pud pages are empty. For this, and instead of introducing another
pud_empty function, consolidate both pte_empty and pmd_empty into
page_empty (the code is actually identical) and use that to also
test the validity of the pud.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2013-08-06 14:05:48 +02:00
|
|
|
}
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
2016-03-23 13:08:02 +01:00
|
|
|
pmd_t *pmd_table __maybe_unused = stage2_pmd_offset(pud, 0);
|
|
|
|
VM_BUG_ON(stage2_pud_huge(*pud));
|
|
|
|
stage2_pud_clear(pud);
|
2014-05-09 23:31:31 +02:00
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2016-03-23 13:08:02 +01:00
|
|
|
stage2_pmd_free(pmd_table);
|
2013-04-12 20:12:05 +02:00
|
|
|
put_page(virt_to_page(pud));
|
|
|
|
}
|
2013-01-21 00:28:06 +01:00
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
|
2013-04-12 20:12:05 +02:00
|
|
|
{
|
2014-05-09 23:31:31 +02:00
|
|
|
pte_t *pte_table = pte_offset_kernel(pmd, 0);
|
2016-03-01 13:00:39 +01:00
|
|
|
VM_BUG_ON(pmd_thp_or_huge(*pmd));
|
2014-05-09 23:31:31 +02:00
|
|
|
pmd_clear(pmd);
|
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
|
|
|
pte_free_kernel(NULL, pte_table);
|
2013-04-12 20:12:05 +02:00
|
|
|
put_page(virt_to_page(pmd));
|
|
|
|
}
|
|
|
|
|
2014-12-19 17:48:06 +01:00
|
|
|
/*
|
|
|
|
* Unmapping vs dcache management:
|
|
|
|
*
|
|
|
|
* If a guest maps certain memory pages as uncached, all writes will
|
|
|
|
* bypass the data cache and go directly to RAM. However, the CPUs
|
|
|
|
* can still speculate reads (not writes) and fill cache lines with
|
|
|
|
* data.
|
|
|
|
*
|
|
|
|
* Those cache lines will be *clean* cache lines though, so a
|
|
|
|
* clean+invalidate operation is equivalent to an invalidate
|
|
|
|
* operation, because no cache lines are marked dirty.
|
|
|
|
*
|
|
|
|
* Those clean cache lines could be filled prior to an uncached write
|
|
|
|
* by the guest, and the cache coherent IO subsystem would therefore
|
|
|
|
* end up writing old data to disk.
|
|
|
|
*
|
|
|
|
* This is why right after unmapping a page/section and invalidating
|
|
|
|
* the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure
|
|
|
|
* the IO subsystem will never hit in the cache.
|
|
|
|
*/
|
2016-03-23 13:08:02 +01:00
|
|
|
static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
|
2014-05-09 23:31:31 +02:00
|
|
|
phys_addr_t addr, phys_addr_t end)
|
2013-04-12 20:12:05 +02:00
|
|
|
{
|
2014-05-09 23:31:31 +02:00
|
|
|
phys_addr_t start_addr = addr;
|
|
|
|
pte_t *pte, *start_pte;
|
|
|
|
|
|
|
|
start_pte = pte = pte_offset_kernel(pmd, addr);
|
|
|
|
do {
|
|
|
|
if (!pte_none(*pte)) {
|
2014-12-19 17:48:06 +01:00
|
|
|
pte_t old_pte = *pte;
|
|
|
|
|
2014-05-09 23:31:31 +02:00
|
|
|
kvm_set_pte(pte, __pte(0));
|
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2014-12-19 17:48:06 +01:00
|
|
|
|
|
|
|
/* No need to invalidate the cache for device mappings */
|
2015-12-03 09:25:22 +01:00
|
|
|
if (!kvm_is_device_pfn(pte_pfn(old_pte)))
|
2014-12-19 17:48:06 +01:00
|
|
|
kvm_flush_dcache_pte(old_pte);
|
|
|
|
|
|
|
|
put_page(virt_to_page(pte));
|
2014-05-09 23:31:31 +02:00
|
|
|
}
|
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
if (stage2_pte_table_empty(start_pte))
|
|
|
|
clear_stage2_pmd_entry(kvm, pmd, start_addr);
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
|
2014-05-09 23:31:31 +02:00
|
|
|
phys_addr_t addr, phys_addr_t end)
|
2013-03-05 03:43:17 +01:00
|
|
|
{
|
2014-05-09 23:31:31 +02:00
|
|
|
phys_addr_t next, start_addr = addr;
|
|
|
|
pmd_t *pmd, *start_pmd;
|
2013-03-05 03:43:17 +01:00
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
start_pmd = pmd = stage2_pmd_offset(pud, addr);
|
2014-05-09 23:31:31 +02:00
|
|
|
do {
|
2016-03-23 13:08:02 +01:00
|
|
|
next = stage2_pmd_addr_end(addr, end);
|
2014-05-09 23:31:31 +02:00
|
|
|
if (!pmd_none(*pmd)) {
|
2016-03-01 13:00:39 +01:00
|
|
|
if (pmd_thp_or_huge(*pmd)) {
|
2014-12-19 17:48:06 +01:00
|
|
|
pmd_t old_pmd = *pmd;
|
|
|
|
|
2014-05-09 23:31:31 +02:00
|
|
|
pmd_clear(pmd);
|
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2014-12-19 17:48:06 +01:00
|
|
|
|
|
|
|
kvm_flush_dcache_pmd(old_pmd);
|
|
|
|
|
2014-05-09 23:31:31 +02:00
|
|
|
put_page(virt_to_page(pmd));
|
|
|
|
} else {
|
2016-03-23 13:08:02 +01:00
|
|
|
unmap_stage2_ptes(kvm, pmd, addr, next);
|
2014-05-09 23:31:31 +02:00
|
|
|
}
|
2012-11-01 17:14:45 +01:00
|
|
|
}
|
2014-05-09 23:31:31 +02:00
|
|
|
} while (pmd++, addr = next, addr != end);
|
2012-11-01 17:14:45 +01:00
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
if (stage2_pmd_table_empty(start_pmd))
|
|
|
|
clear_stage2_pud_entry(kvm, pud, start_addr);
|
2014-05-09 23:31:31 +02:00
|
|
|
}
|
2013-03-05 03:43:17 +01:00
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
|
2014-05-09 23:31:31 +02:00
|
|
|
phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
phys_addr_t next, start_addr = addr;
|
|
|
|
pud_t *pud, *start_pud;
|
2013-04-12 20:12:05 +02:00
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
start_pud = pud = stage2_pud_offset(pgd, addr);
|
2014-05-09 23:31:31 +02:00
|
|
|
do {
|
2016-03-23 13:08:02 +01:00
|
|
|
next = stage2_pud_addr_end(addr, end);
|
|
|
|
if (!stage2_pud_none(*pud)) {
|
|
|
|
if (stage2_pud_huge(*pud)) {
|
2014-12-19 17:48:06 +01:00
|
|
|
pud_t old_pud = *pud;
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
stage2_pud_clear(pud);
|
2014-05-09 23:31:31 +02:00
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2014-12-19 17:48:06 +01:00
|
|
|
kvm_flush_dcache_pud(old_pud);
|
2014-05-09 23:31:31 +02:00
|
|
|
put_page(virt_to_page(pud));
|
|
|
|
} else {
|
2016-03-23 13:08:02 +01:00
|
|
|
unmap_stage2_pmds(kvm, pud, addr, next);
|
2013-04-12 20:12:05 +02:00
|
|
|
}
|
|
|
|
}
|
2014-05-09 23:31:31 +02:00
|
|
|
} while (pud++, addr = next, addr != end);
|
2013-04-12 20:12:05 +02:00
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
if (stage2_pud_table_empty(start_pud))
|
|
|
|
clear_stage2_pgd_entry(kvm, pgd, start_addr);
|
2014-05-09 23:31:31 +02:00
|
|
|
}
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
/**
|
|
|
|
* unmap_stage2_range -- Clear stage2 page table entries to unmap a range
|
|
|
|
* @kvm: The VM pointer
|
|
|
|
* @start: The intermediate physical base address of the range to unmap
|
|
|
|
* @size: The size of the area to unmap
|
|
|
|
*
|
|
|
|
* Clear a range of stage-2 mappings, lowering the various ref-counts. Must
|
|
|
|
* be called while holding mmu_lock (unless for freeing the stage2 pgd before
|
|
|
|
* destroying the VM), otherwise another faulting VCPU may come in and mess
|
|
|
|
* with things behind our backs.
|
|
|
|
*/
|
|
|
|
static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
|
2014-05-09 23:31:31 +02:00
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
phys_addr_t addr = start, end = start + size;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
2016-03-23 13:08:02 +01:00
|
|
|
pgd = kvm->arch.pgd + stage2_pgd_index(addr);
|
2014-05-09 23:31:31 +02:00
|
|
|
do {
|
2016-03-23 13:08:02 +01:00
|
|
|
next = stage2_pgd_addr_end(addr, end);
|
|
|
|
if (!stage2_pgd_none(*pgd))
|
|
|
|
unmap_stage2_puds(kvm, pgd, addr, next);
|
2014-05-09 23:31:31 +02:00
|
|
|
} while (pgd++, addr = next, addr != end);
|
2013-03-05 03:43:17 +01:00
|
|
|
}
|
|
|
|
|
2014-01-15 13:50:23 +01:00
|
|
|
static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
|
|
|
|
phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
|
|
|
|
pte = pte_offset_kernel(pmd, addr);
|
|
|
|
do {
|
2015-12-03 09:25:22 +01:00
|
|
|
if (!pte_none(*pte) && !kvm_is_device_pfn(pte_pfn(*pte)))
|
2014-12-19 17:48:06 +01:00
|
|
|
kvm_flush_dcache_pte(*pte);
|
2014-01-15 13:50:23 +01:00
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
|
|
|
|
phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pmd = stage2_pmd_offset(pud, addr);
|
2014-01-15 13:50:23 +01:00
|
|
|
do {
|
2016-03-22 19:33:45 +01:00
|
|
|
next = stage2_pmd_addr_end(addr, end);
|
2014-01-15 13:50:23 +01:00
|
|
|
if (!pmd_none(*pmd)) {
|
2016-03-01 13:00:39 +01:00
|
|
|
if (pmd_thp_or_huge(*pmd))
|
2014-12-19 17:48:06 +01:00
|
|
|
kvm_flush_dcache_pmd(*pmd);
|
|
|
|
else
|
2014-01-15 13:50:23 +01:00
|
|
|
stage2_flush_ptes(kvm, pmd, addr, next);
|
|
|
|
}
|
|
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
|
|
|
|
phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pud_t *pud;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pud = stage2_pud_offset(pgd, addr);
|
2014-01-15 13:50:23 +01:00
|
|
|
do {
|
2016-03-22 19:33:45 +01:00
|
|
|
next = stage2_pud_addr_end(addr, end);
|
|
|
|
if (!stage2_pud_none(*pud)) {
|
|
|
|
if (stage2_pud_huge(*pud))
|
2014-12-19 17:48:06 +01:00
|
|
|
kvm_flush_dcache_pud(*pud);
|
|
|
|
else
|
2014-01-15 13:50:23 +01:00
|
|
|
stage2_flush_pmds(kvm, pud, addr, next);
|
|
|
|
}
|
|
|
|
} while (pud++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void stage2_flush_memslot(struct kvm *kvm,
|
|
|
|
struct kvm_memory_slot *memslot)
|
|
|
|
{
|
|
|
|
phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
|
|
|
|
phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
|
|
|
|
phys_addr_t next;
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pgd = kvm->arch.pgd + stage2_pgd_index(addr);
|
2014-01-15 13:50:23 +01:00
|
|
|
do {
|
2016-03-22 19:33:45 +01:00
|
|
|
next = stage2_pgd_addr_end(addr, end);
|
2014-01-15 13:50:23 +01:00
|
|
|
stage2_flush_puds(kvm, pgd, addr, next);
|
|
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* stage2_flush_vm - Invalidate cache for pages mapped in stage 2
|
|
|
|
* @kvm: The struct kvm pointer
|
|
|
|
*
|
|
|
|
* Go through the stage 2 page tables and invalidate any cache lines
|
|
|
|
* backing memory already mapped to the VM.
|
|
|
|
*/
|
2014-12-19 17:05:31 +01:00
|
|
|
static void stage2_flush_vm(struct kvm *kvm)
|
2014-01-15 13:50:23 +01:00
|
|
|
{
|
|
|
|
struct kvm_memslots *slots;
|
|
|
|
struct kvm_memory_slot *memslot;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
idx = srcu_read_lock(&kvm->srcu);
|
|
|
|
spin_lock(&kvm->mmu_lock);
|
|
|
|
|
|
|
|
slots = kvm_memslots(kvm);
|
|
|
|
kvm_for_each_memslot(memslot, slots)
|
|
|
|
stage2_flush_memslot(kvm, memslot);
|
|
|
|
|
|
|
|
spin_unlock(&kvm->mmu_lock);
|
|
|
|
srcu_read_unlock(&kvm->srcu, idx);
|
|
|
|
}
|
|
|
|
|
2016-03-22 19:56:21 +01:00
|
|
|
static void clear_hyp_pgd_entry(pgd_t *pgd)
|
|
|
|
{
|
|
|
|
pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL);
|
|
|
|
pgd_clear(pgd);
|
|
|
|
pud_free(NULL, pud_table);
|
|
|
|
put_page(virt_to_page(pgd));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void clear_hyp_pud_entry(pud_t *pud)
|
|
|
|
{
|
|
|
|
pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0);
|
|
|
|
VM_BUG_ON(pud_huge(*pud));
|
|
|
|
pud_clear(pud);
|
|
|
|
pmd_free(NULL, pmd_table);
|
|
|
|
put_page(virt_to_page(pud));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void clear_hyp_pmd_entry(pmd_t *pmd)
|
|
|
|
{
|
|
|
|
pte_t *pte_table = pte_offset_kernel(pmd, 0);
|
|
|
|
VM_BUG_ON(pmd_thp_or_huge(*pmd));
|
|
|
|
pmd_clear(pmd);
|
|
|
|
pte_free_kernel(NULL, pte_table);
|
|
|
|
put_page(virt_to_page(pmd));
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unmap_hyp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pte_t *pte, *start_pte;
|
|
|
|
|
|
|
|
start_pte = pte = pte_offset_kernel(pmd, addr);
|
|
|
|
do {
|
|
|
|
if (!pte_none(*pte)) {
|
|
|
|
kvm_set_pte(pte, __pte(0));
|
|
|
|
put_page(virt_to_page(pte));
|
|
|
|
}
|
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
|
|
|
|
|
|
if (hyp_pte_table_empty(start_pte))
|
|
|
|
clear_hyp_pmd_entry(pmd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unmap_hyp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
phys_addr_t next;
|
|
|
|
pmd_t *pmd, *start_pmd;
|
|
|
|
|
|
|
|
start_pmd = pmd = pmd_offset(pud, addr);
|
|
|
|
do {
|
|
|
|
next = pmd_addr_end(addr, end);
|
|
|
|
/* Hyp doesn't use huge pmds */
|
|
|
|
if (!pmd_none(*pmd))
|
|
|
|
unmap_hyp_ptes(pmd, addr, next);
|
|
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
|
|
|
|
|
if (hyp_pmd_table_empty(start_pmd))
|
|
|
|
clear_hyp_pud_entry(pud);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
phys_addr_t next;
|
|
|
|
pud_t *pud, *start_pud;
|
|
|
|
|
|
|
|
start_pud = pud = pud_offset(pgd, addr);
|
|
|
|
do {
|
|
|
|
next = pud_addr_end(addr, end);
|
|
|
|
/* Hyp doesn't use huge puds */
|
|
|
|
if (!pud_none(*pud))
|
|
|
|
unmap_hyp_pmds(pud, addr, next);
|
|
|
|
} while (pud++, addr = next, addr != end);
|
|
|
|
|
|
|
|
if (hyp_pud_table_empty(start_pud))
|
|
|
|
clear_hyp_pgd_entry(pgd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void unmap_hyp_range(pgd_t *pgdp, phys_addr_t start, u64 size)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
phys_addr_t addr = start, end = start + size;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't unmap anything from HYP, except at the hyp tear down.
|
|
|
|
* Hence, we don't have to invalidate the TLBs here.
|
|
|
|
*/
|
|
|
|
pgd = pgdp + pgd_index(addr);
|
|
|
|
do {
|
|
|
|
next = pgd_addr_end(addr, end);
|
|
|
|
if (!pgd_none(*pgd))
|
|
|
|
unmap_hyp_puds(pgd, addr, next);
|
|
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:06 +01:00
|
|
|
/**
|
2013-04-12 20:12:05 +02:00
|
|
|
* free_hyp_pgds - free Hyp-mode page tables
|
2013-01-21 00:28:06 +01:00
|
|
|
*
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
* Assumes hyp_pgd is a page table used strictly in Hyp-mode and
|
|
|
|
* therefore contains either mappings in the kernel memory area (above
|
|
|
|
* PAGE_OFFSET), or device mappings in the vmalloc range (from
|
|
|
|
* VMALLOC_START to VMALLOC_END).
|
|
|
|
*
|
|
|
|
* boot_hyp_pgd should only map two pages for the init code.
|
2013-01-21 00:28:06 +01:00
|
|
|
*/
|
2013-04-12 20:12:05 +02:00
|
|
|
void free_hyp_pgds(void)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
|
|
|
unsigned long addr;
|
|
|
|
|
2013-04-12 20:12:07 +02:00
|
|
|
mutex_lock(&kvm_hyp_pgd_mutex);
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
|
2016-06-30 19:40:46 +02:00
|
|
|
if (boot_hyp_pgd) {
|
|
|
|
unmap_hyp_range(boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE);
|
|
|
|
free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order);
|
|
|
|
boot_hyp_pgd = NULL;
|
|
|
|
}
|
|
|
|
|
2013-04-12 20:12:05 +02:00
|
|
|
if (hyp_pgd) {
|
2016-06-30 19:40:46 +02:00
|
|
|
unmap_hyp_range(hyp_pgd, hyp_idmap_start, PAGE_SIZE);
|
2013-04-12 20:12:05 +02:00
|
|
|
for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE)
|
2016-06-30 19:40:51 +02:00
|
|
|
unmap_hyp_range(hyp_pgd, kern_hyp_va(addr), PGDIR_SIZE);
|
2013-04-12 20:12:05 +02:00
|
|
|
for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE)
|
2016-06-30 19:40:51 +02:00
|
|
|
unmap_hyp_range(hyp_pgd, kern_hyp_va(addr), PGDIR_SIZE);
|
2013-05-14 13:11:34 +02:00
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
free_pages((unsigned long)hyp_pgd, hyp_pgd_order);
|
2013-04-12 20:12:07 +02:00
|
|
|
hyp_pgd = NULL;
|
2013-04-12 20:12:05 +02:00
|
|
|
}
|
2015-03-19 17:42:28 +01:00
|
|
|
if (merged_hyp_pgd) {
|
|
|
|
clear_page(merged_hyp_pgd);
|
|
|
|
free_page((unsigned long)merged_hyp_pgd);
|
|
|
|
merged_hyp_pgd = NULL;
|
|
|
|
}
|
2013-04-12 20:12:05 +02:00
|
|
|
|
2013-01-21 00:28:06 +01:00
|
|
|
mutex_unlock(&kvm_hyp_pgd_mutex);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
|
2013-04-12 20:12:01 +02:00
|
|
|
unsigned long end, unsigned long pfn,
|
|
|
|
pgprot_t prot)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
unsigned long addr;
|
|
|
|
|
2013-04-12 20:12:02 +02:00
|
|
|
addr = start;
|
|
|
|
do {
|
2013-04-12 20:12:01 +02:00
|
|
|
pte = pte_offset_kernel(pmd, addr);
|
|
|
|
kvm_set_pte(pte, pfn_pte(pfn, prot));
|
2013-04-12 20:12:05 +02:00
|
|
|
get_page(virt_to_page(pte));
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
kvm_flush_dcache_to_poc(pte, sizeof(*pte));
|
2013-04-12 20:12:01 +02:00
|
|
|
pfn++;
|
2013-04-12 20:12:02 +02:00
|
|
|
} while (addr += PAGE_SIZE, addr != end);
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
|
2013-04-12 20:12:01 +02:00
|
|
|
unsigned long end, unsigned long pfn,
|
|
|
|
pgprot_t prot)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
|
|
|
unsigned long addr, next;
|
|
|
|
|
2013-04-12 20:12:02 +02:00
|
|
|
addr = start;
|
|
|
|
do {
|
2013-04-12 20:12:01 +02:00
|
|
|
pmd = pmd_offset(pud, addr);
|
2013-01-21 00:28:06 +01:00
|
|
|
|
|
|
|
BUG_ON(pmd_sect(*pmd));
|
|
|
|
|
|
|
|
if (pmd_none(*pmd)) {
|
2013-04-12 20:12:01 +02:00
|
|
|
pte = pte_alloc_one_kernel(NULL, addr);
|
2013-01-21 00:28:06 +01:00
|
|
|
if (!pte) {
|
|
|
|
kvm_err("Cannot allocate Hyp pte\n");
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
pmd_populate_kernel(NULL, pmd, pte);
|
2013-04-12 20:12:05 +02:00
|
|
|
get_page(virt_to_page(pmd));
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
kvm_flush_dcache_to_poc(pmd, sizeof(*pmd));
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
next = pmd_addr_end(addr, end);
|
|
|
|
|
2013-04-12 20:12:01 +02:00
|
|
|
create_hyp_pte_mappings(pmd, addr, next, pfn, prot);
|
|
|
|
pfn += (next - addr) >> PAGE_SHIFT;
|
2013-04-12 20:12:02 +02:00
|
|
|
} while (addr = next, addr != end);
|
2013-01-21 00:28:06 +01:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
|
|
|
|
unsigned long end, unsigned long pfn,
|
|
|
|
pgprot_t prot)
|
|
|
|
{
|
|
|
|
pud_t *pud;
|
|
|
|
pmd_t *pmd;
|
|
|
|
unsigned long addr, next;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
addr = start;
|
|
|
|
do {
|
|
|
|
pud = pud_offset(pgd, addr);
|
|
|
|
|
|
|
|
if (pud_none_or_clear_bad(pud)) {
|
|
|
|
pmd = pmd_alloc_one(NULL, addr);
|
|
|
|
if (!pmd) {
|
|
|
|
kvm_err("Cannot allocate Hyp pmd\n");
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
pud_populate(NULL, pud, pmd);
|
|
|
|
get_page(virt_to_page(pud));
|
|
|
|
kvm_flush_dcache_to_poc(pud, sizeof(*pud));
|
|
|
|
}
|
|
|
|
|
|
|
|
next = pud_addr_end(addr, end);
|
|
|
|
ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
pfn += (next - addr) >> PAGE_SHIFT;
|
|
|
|
} while (addr = next, addr != end);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-04-12 20:12:01 +02:00
|
|
|
static int __create_hyp_mappings(pgd_t *pgdp,
|
|
|
|
unsigned long start, unsigned long end,
|
|
|
|
unsigned long pfn, pgprot_t prot)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
pud_t *pud;
|
|
|
|
unsigned long addr, next;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
mutex_lock(&kvm_hyp_pgd_mutex);
|
2013-04-12 20:12:02 +02:00
|
|
|
addr = start & PAGE_MASK;
|
|
|
|
end = PAGE_ALIGN(end);
|
|
|
|
do {
|
2013-04-12 20:12:01 +02:00
|
|
|
pgd = pgdp + pgd_index(addr);
|
2013-01-21 00:28:06 +01:00
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
if (pgd_none(*pgd)) {
|
|
|
|
pud = pud_alloc_one(NULL, addr);
|
|
|
|
if (!pud) {
|
|
|
|
kvm_err("Cannot allocate Hyp pud\n");
|
2013-01-21 00:28:06 +01:00
|
|
|
err = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
2014-10-10 12:14:28 +02:00
|
|
|
pgd_populate(NULL, pgd, pud);
|
|
|
|
get_page(virt_to_page(pgd));
|
|
|
|
kvm_flush_dcache_to_poc(pgd, sizeof(*pgd));
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
next = pgd_addr_end(addr, end);
|
2014-10-10 12:14:28 +02:00
|
|
|
err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);
|
2013-01-21 00:28:06 +01:00
|
|
|
if (err)
|
|
|
|
goto out;
|
2013-04-12 20:12:01 +02:00
|
|
|
pfn += (next - addr) >> PAGE_SHIFT;
|
2013-04-12 20:12:02 +02:00
|
|
|
} while (addr = next, addr != end);
|
2013-01-21 00:28:06 +01:00
|
|
|
out:
|
|
|
|
mutex_unlock(&kvm_hyp_pgd_mutex);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2013-11-15 22:14:12 +01:00
|
|
|
static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
|
|
|
|
{
|
|
|
|
if (!is_vmalloc_addr(kaddr)) {
|
|
|
|
BUG_ON(!virt_addr_valid(kaddr));
|
|
|
|
return __pa(kaddr);
|
|
|
|
} else {
|
|
|
|
return page_to_phys(vmalloc_to_page(kaddr)) +
|
|
|
|
offset_in_page(kaddr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:06 +01:00
|
|
|
/**
|
2012-10-28 02:09:14 +02:00
|
|
|
* create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
|
2013-01-21 00:28:06 +01:00
|
|
|
* @from: The virtual kernel start address of the range
|
|
|
|
* @to: The virtual kernel end address of the range (exclusive)
|
2016-06-13 16:00:45 +02:00
|
|
|
* @prot: The protection to be applied to this range
|
2013-01-21 00:28:06 +01:00
|
|
|
*
|
2012-10-28 02:09:14 +02:00
|
|
|
* The same virtual address as the kernel virtual address is also used
|
|
|
|
* in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
|
|
|
|
* physical pages.
|
2013-01-21 00:28:06 +01:00
|
|
|
*/
|
2016-06-13 16:00:45 +02:00
|
|
|
int create_hyp_mappings(void *from, void *to, pgprot_t prot)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
2013-11-15 22:14:12 +01:00
|
|
|
phys_addr_t phys_addr;
|
|
|
|
unsigned long virt_addr;
|
2016-06-30 19:40:51 +02:00
|
|
|
unsigned long start = kern_hyp_va((unsigned long)from);
|
|
|
|
unsigned long end = kern_hyp_va((unsigned long)to);
|
2013-04-12 20:12:01 +02:00
|
|
|
|
2015-01-29 12:59:54 +01:00
|
|
|
if (is_kernel_in_hyp_mode())
|
|
|
|
return 0;
|
|
|
|
|
2013-11-15 22:14:12 +01:00
|
|
|
start = start & PAGE_MASK;
|
|
|
|
end = PAGE_ALIGN(end);
|
2013-04-12 20:12:01 +02:00
|
|
|
|
2013-11-15 22:14:12 +01:00
|
|
|
for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
|
|
|
|
int err;
|
2013-04-12 20:12:01 +02:00
|
|
|
|
2013-11-15 22:14:12 +01:00
|
|
|
phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
|
|
|
|
err = __create_hyp_mappings(hyp_pgd, virt_addr,
|
|
|
|
virt_addr + PAGE_SIZE,
|
|
|
|
__phys_to_pfn(phys_addr),
|
2016-06-13 16:00:45 +02:00
|
|
|
prot);
|
2013-11-15 22:14:12 +01:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2012-10-28 02:09:14 +02:00
|
|
|
* create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode
|
|
|
|
* @from: The kernel start VA of the range
|
|
|
|
* @to: The kernel end VA of the range (exclusive)
|
2013-04-12 20:12:01 +02:00
|
|
|
* @phys_addr: The physical start address which gets mapped
|
2012-10-28 02:09:14 +02:00
|
|
|
*
|
|
|
|
* The resulting HYP VA is the same as the kernel VA, modulo
|
|
|
|
* HYP_PAGE_OFFSET.
|
2013-01-21 00:28:06 +01:00
|
|
|
*/
|
2013-04-12 20:12:01 +02:00
|
|
|
int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
|
2013-01-21 00:28:06 +01:00
|
|
|
{
|
2016-06-30 19:40:51 +02:00
|
|
|
unsigned long start = kern_hyp_va((unsigned long)from);
|
|
|
|
unsigned long end = kern_hyp_va((unsigned long)to);
|
2013-04-12 20:12:01 +02:00
|
|
|
|
2015-01-29 12:59:54 +01:00
|
|
|
if (is_kernel_in_hyp_mode())
|
|
|
|
return 0;
|
|
|
|
|
2013-04-12 20:12:01 +02:00
|
|
|
/* Check for a valid kernel IO mapping */
|
|
|
|
if (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return __create_hyp_mappings(hyp_pgd, start, end,
|
|
|
|
__phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:07 +01:00
|
|
|
/**
|
|
|
|
* kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
|
|
|
|
* @kvm: The KVM struct pointer for the VM.
|
|
|
|
*
|
2015-11-16 12:28:16 +01:00
|
|
|
* Allocates only the stage-2 HW PGD level table(s) (can support either full
|
|
|
|
* 40-bit input addresses or limited to 32-bit input addresses). Clears the
|
|
|
|
* allocated pages.
|
2013-01-21 00:28:07 +01:00
|
|
|
*
|
|
|
|
* Note we don't need locking here as this is only called when the VM is
|
|
|
|
* created, which can only be done once.
|
|
|
|
*/
|
|
|
|
int kvm_alloc_stage2_pgd(struct kvm *kvm)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
|
|
if (kvm->arch.pgd != NULL) {
|
|
|
|
kvm_err("kvm_arch already initialized?\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2016-03-22 18:01:21 +01:00
|
|
|
/* Allocate the HW PGD, making sure that each page gets its own refcount */
|
|
|
|
pgd = alloc_pages_exact(S2_PGD_SIZE, GFP_KERNEL | __GFP_ZERO);
|
|
|
|
if (!pgd)
|
2015-03-10 20:06:59 +01:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2013-01-21 00:28:07 +01:00
|
|
|
kvm->arch.pgd = pgd;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-11-27 10:35:03 +01:00
|
|
|
static void stage2_unmap_memslot(struct kvm *kvm,
|
|
|
|
struct kvm_memory_slot *memslot)
|
|
|
|
{
|
|
|
|
hva_t hva = memslot->userspace_addr;
|
|
|
|
phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
|
|
|
|
phys_addr_t size = PAGE_SIZE * memslot->npages;
|
|
|
|
hva_t reg_end = hva + size;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A memory region could potentially cover multiple VMAs, and any holes
|
|
|
|
* between them, so iterate over all of them to find out if we should
|
|
|
|
* unmap any of them.
|
|
|
|
*
|
|
|
|
* +--------------------------------------------+
|
|
|
|
* +---------------+----------------+ +----------------+
|
|
|
|
* | : VMA 1 | VMA 2 | | VMA 3 : |
|
|
|
|
* +---------------+----------------+ +----------------+
|
|
|
|
* | memory region |
|
|
|
|
* +--------------------------------------------+
|
|
|
|
*/
|
|
|
|
do {
|
|
|
|
struct vm_area_struct *vma = find_vma(current->mm, hva);
|
|
|
|
hva_t vm_start, vm_end;
|
|
|
|
|
|
|
|
if (!vma || vma->vm_start >= reg_end)
|
|
|
|
break;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Take the intersection of this VMA with the memory region
|
|
|
|
*/
|
|
|
|
vm_start = max(hva, vma->vm_start);
|
|
|
|
vm_end = min(reg_end, vma->vm_end);
|
|
|
|
|
|
|
|
if (!(vma->vm_flags & VM_PFNMAP)) {
|
|
|
|
gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
|
|
|
|
unmap_stage2_range(kvm, gpa, vm_end - vm_start);
|
|
|
|
}
|
|
|
|
hva = vm_end;
|
|
|
|
} while (hva < reg_end);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* stage2_unmap_vm - Unmap Stage-2 RAM mappings
|
|
|
|
* @kvm: The struct kvm pointer
|
|
|
|
*
|
|
|
|
* Go through the memregions and unmap any reguler RAM
|
|
|
|
* backing memory already mapped to the VM.
|
|
|
|
*/
|
|
|
|
void stage2_unmap_vm(struct kvm *kvm)
|
|
|
|
{
|
|
|
|
struct kvm_memslots *slots;
|
|
|
|
struct kvm_memory_slot *memslot;
|
|
|
|
int idx;
|
|
|
|
|
|
|
|
idx = srcu_read_lock(&kvm->srcu);
|
|
|
|
spin_lock(&kvm->mmu_lock);
|
|
|
|
|
|
|
|
slots = kvm_memslots(kvm);
|
|
|
|
kvm_for_each_memslot(memslot, slots)
|
|
|
|
stage2_unmap_memslot(kvm, memslot);
|
|
|
|
|
|
|
|
spin_unlock(&kvm->mmu_lock);
|
|
|
|
srcu_read_unlock(&kvm->srcu, idx);
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:07 +01:00
|
|
|
/**
|
|
|
|
* kvm_free_stage2_pgd - free all stage-2 tables
|
|
|
|
* @kvm: The KVM struct pointer for the VM.
|
|
|
|
*
|
|
|
|
* Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
|
|
|
|
* underlying level-2 and level-3 tables before freeing the actual level-1 table
|
|
|
|
* and setting the struct pointer to NULL.
|
|
|
|
*
|
|
|
|
* Note we don't need locking here as this is only called when the VM is
|
|
|
|
* destroyed, which can only be done once.
|
|
|
|
*/
|
|
|
|
void kvm_free_stage2_pgd(struct kvm *kvm)
|
|
|
|
{
|
|
|
|
if (kvm->arch.pgd == NULL)
|
|
|
|
return;
|
|
|
|
|
|
|
|
unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
|
2016-03-22 18:01:21 +01:00
|
|
|
/* Free the HW pgd, one page at a time */
|
|
|
|
free_pages_exact(kvm->arch.pgd, S2_PGD_SIZE);
|
2013-01-21 00:28:07 +01:00
|
|
|
kvm->arch.pgd = NULL;
|
|
|
|
}
|
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
|
2012-11-01 17:14:45 +01:00
|
|
|
phys_addr_t addr)
|
2013-01-21 00:28:07 +01:00
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
pud_t *pud;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pgd = kvm->arch.pgd + stage2_pgd_index(addr);
|
|
|
|
if (WARN_ON(stage2_pgd_none(*pgd))) {
|
2014-10-10 12:14:28 +02:00
|
|
|
if (!cache)
|
|
|
|
return NULL;
|
|
|
|
pud = mmu_memory_cache_alloc(cache);
|
2016-03-22 19:33:45 +01:00
|
|
|
stage2_pgd_populate(pgd, pud);
|
2014-10-10 12:14:28 +02:00
|
|
|
get_page(virt_to_page(pgd));
|
|
|
|
}
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
return stage2_pud_offset(pgd, addr);
|
2014-10-10 12:14:28 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
|
|
|
|
phys_addr_t addr)
|
|
|
|
{
|
|
|
|
pud_t *pud;
|
|
|
|
pmd_t *pmd;
|
|
|
|
|
|
|
|
pud = stage2_get_pud(kvm, cache, addr);
|
2016-03-22 19:33:45 +01:00
|
|
|
if (stage2_pud_none(*pud)) {
|
2013-01-21 00:28:07 +01:00
|
|
|
if (!cache)
|
2012-11-01 17:14:45 +01:00
|
|
|
return NULL;
|
2013-01-21 00:28:07 +01:00
|
|
|
pmd = mmu_memory_cache_alloc(cache);
|
2016-03-22 19:33:45 +01:00
|
|
|
stage2_pud_populate(pud, pmd);
|
2013-01-21 00:28:07 +01:00
|
|
|
get_page(virt_to_page(pud));
|
2012-10-15 12:27:37 +02:00
|
|
|
}
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
return stage2_pmd_offset(pud, addr);
|
2012-11-01 17:14:45 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
|
|
|
|
*cache, phys_addr_t addr, const pmd_t *new_pmd)
|
|
|
|
{
|
|
|
|
pmd_t *pmd, old_pmd;
|
|
|
|
|
|
|
|
pmd = stage2_get_pmd(kvm, cache, addr);
|
|
|
|
VM_BUG_ON(!pmd);
|
2013-01-21 00:28:07 +01:00
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
/*
|
|
|
|
* Mapping in huge pages should only happen through a fault. If a
|
|
|
|
* page is merged into a transparent huge page, the individual
|
|
|
|
* subpages of that huge page should be unmapped through MMU
|
|
|
|
* notifiers before we get here.
|
|
|
|
*
|
|
|
|
* Merging of CompoundPages is not supported; they should become
|
|
|
|
* splitting first, unmapped, merged, and mapped back in on-demand.
|
|
|
|
*/
|
|
|
|
VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd));
|
|
|
|
|
|
|
|
old_pmd = *pmd;
|
2016-04-28 17:16:31 +02:00
|
|
|
if (pmd_present(old_pmd)) {
|
|
|
|
pmd_clear(pmd);
|
2012-11-01 17:14:45 +01:00
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2016-04-28 17:16:31 +02:00
|
|
|
} else {
|
2012-11-01 17:14:45 +01:00
|
|
|
get_page(virt_to_page(pmd));
|
2016-04-28 17:16:31 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
kvm_set_pmd(pmd, *new_pmd);
|
2012-11-01 17:14:45 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
|
2015-01-16 00:58:58 +01:00
|
|
|
phys_addr_t addr, const pte_t *new_pte,
|
|
|
|
unsigned long flags)
|
2012-11-01 17:14:45 +01:00
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte, old_pte;
|
2015-01-16 00:58:58 +01:00
|
|
|
bool iomap = flags & KVM_S2PTE_FLAG_IS_IOMAP;
|
|
|
|
bool logging_active = flags & KVM_S2_FLAG_LOGGING_ACTIVE;
|
|
|
|
|
|
|
|
VM_BUG_ON(logging_active && !cache);
|
2012-11-01 17:14:45 +01:00
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
/* Create stage-2 page table mapping - Levels 0 and 1 */
|
2012-11-01 17:14:45 +01:00
|
|
|
pmd = stage2_get_pmd(kvm, cache, addr);
|
|
|
|
if (!pmd) {
|
|
|
|
/*
|
|
|
|
* Ignore calls from kvm_set_spte_hva for unallocated
|
|
|
|
* address ranges.
|
|
|
|
*/
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
/*
|
|
|
|
* While dirty page logging - dissolve huge PMD, then continue on to
|
|
|
|
* allocate page.
|
|
|
|
*/
|
|
|
|
if (logging_active)
|
|
|
|
stage2_dissolve_pmd(kvm, addr, pmd);
|
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
/* Create stage-2 page mappings - Level 2 */
|
2013-01-21 00:28:07 +01:00
|
|
|
if (pmd_none(*pmd)) {
|
|
|
|
if (!cache)
|
|
|
|
return 0; /* ignore calls from kvm_set_spte_hva */
|
|
|
|
pte = mmu_memory_cache_alloc(cache);
|
|
|
|
pmd_populate_kernel(NULL, pmd, pte);
|
|
|
|
get_page(virt_to_page(pmd));
|
2012-10-15 12:27:37 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
pte = pte_offset_kernel(pmd, addr);
|
2013-01-21 00:28:07 +01:00
|
|
|
|
|
|
|
if (iomap && pte_present(*pte))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
/* Create 2nd stage page table mapping - Level 3 */
|
|
|
|
old_pte = *pte;
|
2016-04-28 17:16:31 +02:00
|
|
|
if (pte_present(old_pte)) {
|
|
|
|
kvm_set_pte(pte, __pte(0));
|
2013-01-28 16:27:00 +01:00
|
|
|
kvm_tlb_flush_vmid_ipa(kvm, addr);
|
2016-04-28 17:16:31 +02:00
|
|
|
} else {
|
2013-01-21 00:28:07 +01:00
|
|
|
get_page(virt_to_page(pte));
|
2016-04-28 17:16:31 +02:00
|
|
|
}
|
2013-01-21 00:28:07 +01:00
|
|
|
|
2016-04-28 17:16:31 +02:00
|
|
|
kvm_set_pte(pte, *new_pte);
|
2013-01-21 00:28:07 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-04-13 18:57:37 +02:00
|
|
|
#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
|
|
|
|
static int stage2_ptep_test_and_clear_young(pte_t *pte)
|
|
|
|
{
|
|
|
|
if (pte_young(*pte)) {
|
|
|
|
*pte = pte_mkold(*pte);
|
|
|
|
return 1;
|
|
|
|
}
|
2013-01-21 00:28:07 +01:00
|
|
|
return 0;
|
|
|
|
}
|
2016-04-13 18:57:37 +02:00
|
|
|
#else
|
|
|
|
static int stage2_ptep_test_and_clear_young(pte_t *pte)
|
|
|
|
{
|
|
|
|
return __ptep_test_and_clear_young(pte);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static int stage2_pmdp_test_and_clear_young(pmd_t *pmd)
|
|
|
|
{
|
|
|
|
return stage2_ptep_test_and_clear_young((pte_t *)pmd);
|
|
|
|
}
|
2013-01-21 00:28:07 +01:00
|
|
|
|
|
|
|
/**
|
|
|
|
* kvm_phys_addr_ioremap - map a device range to guest IPA
|
|
|
|
*
|
|
|
|
* @kvm: The KVM pointer
|
|
|
|
* @guest_ipa: The IPA at which to insert the mapping
|
|
|
|
* @pa: The physical address of the device
|
|
|
|
* @size: The size of the mapping
|
|
|
|
*/
|
|
|
|
int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
|
2014-09-17 23:56:18 +02:00
|
|
|
phys_addr_t pa, unsigned long size, bool writable)
|
2013-01-21 00:28:07 +01:00
|
|
|
{
|
|
|
|
phys_addr_t addr, end;
|
|
|
|
int ret = 0;
|
|
|
|
unsigned long pfn;
|
|
|
|
struct kvm_mmu_memory_cache cache = { 0, };
|
|
|
|
|
|
|
|
end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
|
|
|
|
pfn = __phys_to_pfn(pa);
|
|
|
|
|
|
|
|
for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
|
2012-10-15 12:27:37 +02:00
|
|
|
pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE);
|
2013-01-21 00:28:07 +01:00
|
|
|
|
2014-09-17 23:56:18 +02:00
|
|
|
if (writable)
|
2016-04-13 18:57:37 +02:00
|
|
|
pte = kvm_s2pte_mkwrite(pte);
|
2014-09-17 23:56:18 +02:00
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
ret = mmu_topup_memory_cache(&cache, KVM_MMU_CACHE_MIN_PAGES,
|
|
|
|
KVM_NR_MEM_OBJS);
|
2013-01-21 00:28:07 +01:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
spin_lock(&kvm->mmu_lock);
|
2015-01-16 00:58:58 +01:00
|
|
|
ret = stage2_set_pte(kvm, &cache, addr, &pte,
|
|
|
|
KVM_S2PTE_FLAG_IS_IOMAP);
|
2013-01-21 00:28:07 +01:00
|
|
|
spin_unlock(&kvm->mmu_lock);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
pfn++;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
mmu_free_memory_cache(&cache);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
kvm: rename pfn_t to kvm_pfn_t
To date, we have implemented two I/O usage models for persistent memory,
PMEM (a persistent "ram disk") and DAX (mmap persistent memory into
userspace). This series adds a third, DAX-GUP, that allows DAX mappings
to be the target of direct-i/o. It allows userspace to coordinate
DMA/RDMA from/to persistent memory.
The implementation leverages the ZONE_DEVICE mm-zone that went into
4.3-rc1 (also discussed at kernel summit) to flag pages that are owned
and dynamically mapped by a device driver. The pmem driver, after
mapping a persistent memory range into the system memmap via
devm_memremap_pages(), arranges for DAX to distinguish pfn-only versus
page-backed pmem-pfns via flags in the new pfn_t type.
The DAX code, upon seeing a PFN_DEV+PFN_MAP flagged pfn, flags the
resulting pte(s) inserted into the process page tables with a new
_PAGE_DEVMAP flag. Later, when get_user_pages() is walking ptes it keys
off _PAGE_DEVMAP to pin the device hosting the page range active.
Finally, get_page() and put_page() are modified to take references
against the device driver established page mapping.
Finally, this need for "struct page" for persistent memory requires
memory capacity to store the memmap array. Given the memmap array for a
large pool of persistent may exhaust available DRAM introduce a
mechanism to allocate the memmap from persistent memory. The new
"struct vmem_altmap *" parameter to devm_memremap_pages() enables
arch_add_memory() to use reserved pmem capacity rather than the page
allocator.
This patch (of 18):
The core has developed a need for a "pfn_t" type [1]. Move the existing
pfn_t in KVM to kvm_pfn_t [2].
[1]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002199.html
[2]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002218.html
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-16 01:56:11 +01:00
|
|
|
static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap)
|
2013-10-03 00:32:01 +02:00
|
|
|
{
|
kvm: rename pfn_t to kvm_pfn_t
To date, we have implemented two I/O usage models for persistent memory,
PMEM (a persistent "ram disk") and DAX (mmap persistent memory into
userspace). This series adds a third, DAX-GUP, that allows DAX mappings
to be the target of direct-i/o. It allows userspace to coordinate
DMA/RDMA from/to persistent memory.
The implementation leverages the ZONE_DEVICE mm-zone that went into
4.3-rc1 (also discussed at kernel summit) to flag pages that are owned
and dynamically mapped by a device driver. The pmem driver, after
mapping a persistent memory range into the system memmap via
devm_memremap_pages(), arranges for DAX to distinguish pfn-only versus
page-backed pmem-pfns via flags in the new pfn_t type.
The DAX code, upon seeing a PFN_DEV+PFN_MAP flagged pfn, flags the
resulting pte(s) inserted into the process page tables with a new
_PAGE_DEVMAP flag. Later, when get_user_pages() is walking ptes it keys
off _PAGE_DEVMAP to pin the device hosting the page range active.
Finally, get_page() and put_page() are modified to take references
against the device driver established page mapping.
Finally, this need for "struct page" for persistent memory requires
memory capacity to store the memmap array. Given the memmap array for a
large pool of persistent may exhaust available DRAM introduce a
mechanism to allocate the memmap from persistent memory. The new
"struct vmem_altmap *" parameter to devm_memremap_pages() enables
arch_add_memory() to use reserved pmem capacity rather than the page
allocator.
This patch (of 18):
The core has developed a need for a "pfn_t" type [1]. Move the existing
pfn_t in KVM to kvm_pfn_t [2].
[1]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002199.html
[2]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002218.html
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-16 01:56:11 +01:00
|
|
|
kvm_pfn_t pfn = *pfnp;
|
2013-10-03 00:32:01 +02:00
|
|
|
gfn_t gfn = *ipap >> PAGE_SHIFT;
|
|
|
|
|
mm: thp: kvm: fix memory corruption in KVM with THP enabled
After the THP refcounting change, obtaining a compound pages from
get_user_pages() no longer allows us to assume the entire compound page
is immediately mappable from a secondary MMU.
A secondary MMU doesn't want to call get_user_pages() more than once for
each compound page, in order to know if it can map the whole compound
page. So a secondary MMU needs to know from a single get_user_pages()
invocation when it can map immediately the entire compound page to avoid
a flood of unnecessary secondary MMU faults and spurious
atomic_inc()/atomic_dec() (pages don't have to be pinned by MMU notifier
users).
Ideally instead of the page->_mapcount < 1 check, get_user_pages()
should return the granularity of the "page" mapping in the "mm" passed
to get_user_pages(). However it's non trivial change to pass the "pmd"
status belonging to the "mm" walked by get_user_pages up the stack (up
to the caller of get_user_pages). So the fix just checks if there is
not a single pte mapping on the page returned by get_user_pages, and in
turn if the caller can assume that the whole compound page is mapped in
the current "mm" (in a pmd_trans_huge()). In such case the entire
compound page is safe to map into the secondary MMU without additional
get_user_pages() calls on the surrounding tail/head pages. In addition
of being faster, not having to run other get_user_pages() calls also
reduces the memory footprint of the secondary MMU fault in case the pmd
split happened as result of memory pressure.
Without this fix after a MADV_DONTNEED (like invoked by QEMU during
postcopy live migration or balloning) or after generic swapping (with a
failure in split_huge_page() that would only result in pmd splitting and
not a physical page split), KVM would map the whole compound page into
the shadow pagetables, despite regular faults or userfaults (like
UFFDIO_COPY) may map regular pages into the primary MMU as result of the
pte faults, leading to the guest mode and userland mode going out of
sync and not working on the same memory at all times.
Any other secondary MMU notifier manager (KVM is just one of the many
MMU notifier users) will need the same information if it doesn't want to
run a flood of get_user_pages_fast and it can support multiple
granularity in the secondary MMU mappings, so I think it is justified to
be exposed not just to KVM.
The other option would be to move transparent_hugepage_adjust to
mm/huge_memory.c but that currently has all kind of KVM data structures
in it, so it's definitely not a cut-and-paste work, so I couldn't do a
fix as cleaner as this one for 4.6.
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: "Li, Liang Z" <liang.z.li@intel.com>
Cc: Amit Shah <amit.shah@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-05-06 01:22:20 +02:00
|
|
|
if (PageTransCompoundMap(pfn_to_page(pfn))) {
|
2013-10-03 00:32:01 +02:00
|
|
|
unsigned long mask;
|
|
|
|
/*
|
|
|
|
* The address we faulted on is backed by a transparent huge
|
|
|
|
* page. However, because we map the compound huge page and
|
|
|
|
* not the individual tail page, we need to transfer the
|
|
|
|
* refcount to the head page. We have to be careful that the
|
|
|
|
* THP doesn't start to split while we are adjusting the
|
|
|
|
* refcounts.
|
|
|
|
*
|
|
|
|
* We are sure this doesn't happen, because mmu_notifier_retry
|
|
|
|
* was successful and we are holding the mmu_lock, so if this
|
|
|
|
* THP is trying to split, it will be blocked in the mmu
|
|
|
|
* notifier before touching any of the pages, specifically
|
|
|
|
* before being able to call __split_huge_page_refcount().
|
|
|
|
*
|
|
|
|
* We can therefore safely transfer the refcount from PG_tail
|
|
|
|
* to PG_head and switch the pfn from a tail page to the head
|
|
|
|
* page accordingly.
|
|
|
|
*/
|
|
|
|
mask = PTRS_PER_PMD - 1;
|
|
|
|
VM_BUG_ON((gfn & mask) != (pfn & mask));
|
|
|
|
if (pfn & mask) {
|
|
|
|
*ipap &= PMD_MASK;
|
|
|
|
kvm_release_pfn_clean(pfn);
|
|
|
|
pfn &= ~mask;
|
|
|
|
kvm_get_pfn(pfn);
|
|
|
|
*pfnp = pfn;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2014-09-09 12:27:09 +02:00
|
|
|
static bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
if (kvm_vcpu_trap_is_iabt(vcpu))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return kvm_vcpu_dabt_iswrite(vcpu);
|
|
|
|
}
|
|
|
|
|
2015-01-16 00:58:56 +01:00
|
|
|
/**
|
|
|
|
* stage2_wp_ptes - write protect PMD range
|
|
|
|
* @pmd: pointer to pmd entry
|
|
|
|
* @addr: range start address
|
|
|
|
* @end: range end address
|
|
|
|
*/
|
|
|
|
static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pte_t *pte;
|
|
|
|
|
|
|
|
pte = pte_offset_kernel(pmd, addr);
|
|
|
|
do {
|
|
|
|
if (!pte_none(*pte)) {
|
|
|
|
if (!kvm_s2pte_readonly(pte))
|
|
|
|
kvm_set_s2pte_readonly(pte);
|
|
|
|
}
|
|
|
|
} while (pte++, addr += PAGE_SIZE, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* stage2_wp_pmds - write protect PUD range
|
|
|
|
* @pud: pointer to pud entry
|
|
|
|
* @addr: range start address
|
|
|
|
* @end: range end address
|
|
|
|
*/
|
|
|
|
static void stage2_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pmd = stage2_pmd_offset(pud, addr);
|
2015-01-16 00:58:56 +01:00
|
|
|
|
|
|
|
do {
|
2016-03-22 19:33:45 +01:00
|
|
|
next = stage2_pmd_addr_end(addr, end);
|
2015-01-16 00:58:56 +01:00
|
|
|
if (!pmd_none(*pmd)) {
|
2016-03-01 13:00:39 +01:00
|
|
|
if (pmd_thp_or_huge(*pmd)) {
|
2015-01-16 00:58:56 +01:00
|
|
|
if (!kvm_s2pmd_readonly(pmd))
|
|
|
|
kvm_set_s2pmd_readonly(pmd);
|
|
|
|
} else {
|
|
|
|
stage2_wp_ptes(pmd, addr, next);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} while (pmd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* stage2_wp_puds - write protect PGD range
|
|
|
|
* @pgd: pointer to pgd entry
|
|
|
|
* @addr: range start address
|
|
|
|
* @end: range end address
|
|
|
|
*
|
|
|
|
* Process PUD entries, for a huge PUD we cause a panic.
|
|
|
|
*/
|
|
|
|
static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pud_t *pud;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pud = stage2_pud_offset(pgd, addr);
|
2015-01-16 00:58:56 +01:00
|
|
|
do {
|
2016-03-22 19:33:45 +01:00
|
|
|
next = stage2_pud_addr_end(addr, end);
|
|
|
|
if (!stage2_pud_none(*pud)) {
|
2015-01-16 00:58:56 +01:00
|
|
|
/* TODO:PUD not supported, revisit later if supported */
|
2016-03-22 19:33:45 +01:00
|
|
|
BUG_ON(stage2_pud_huge(*pud));
|
2015-01-16 00:58:56 +01:00
|
|
|
stage2_wp_pmds(pud, addr, next);
|
|
|
|
}
|
|
|
|
} while (pud++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* stage2_wp_range() - write protect stage2 memory region range
|
|
|
|
* @kvm: The KVM pointer
|
|
|
|
* @addr: Start address of range
|
|
|
|
* @end: End address of range
|
|
|
|
*/
|
|
|
|
static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
|
|
|
|
{
|
|
|
|
pgd_t *pgd;
|
|
|
|
phys_addr_t next;
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
pgd = kvm->arch.pgd + stage2_pgd_index(addr);
|
2015-01-16 00:58:56 +01:00
|
|
|
do {
|
|
|
|
/*
|
|
|
|
* Release kvm_mmu_lock periodically if the memory region is
|
|
|
|
* large. Otherwise, we may see kernel panics with
|
2015-01-23 10:49:31 +01:00
|
|
|
* CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR,
|
|
|
|
* CONFIG_LOCKDEP. Additionally, holding the lock too long
|
2015-01-16 00:58:56 +01:00
|
|
|
* will also starve other vCPUs.
|
|
|
|
*/
|
|
|
|
if (need_resched() || spin_needbreak(&kvm->mmu_lock))
|
|
|
|
cond_resched_lock(&kvm->mmu_lock);
|
|
|
|
|
2016-03-22 19:33:45 +01:00
|
|
|
next = stage2_pgd_addr_end(addr, end);
|
|
|
|
if (stage2_pgd_present(*pgd))
|
2015-01-16 00:58:56 +01:00
|
|
|
stage2_wp_puds(pgd, addr, next);
|
|
|
|
} while (pgd++, addr = next, addr != end);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot
|
|
|
|
* @kvm: The KVM pointer
|
|
|
|
* @slot: The memory slot to write protect
|
|
|
|
*
|
|
|
|
* Called to start logging dirty pages after memory region
|
|
|
|
* KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns
|
|
|
|
* all present PMD and PTEs are write protected in the memory region.
|
|
|
|
* Afterwards read of dirty page log can be called.
|
|
|
|
*
|
|
|
|
* Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired,
|
|
|
|
* serializing operations for VM memory regions.
|
|
|
|
*/
|
|
|
|
void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
|
|
|
|
{
|
2015-05-17 16:20:07 +02:00
|
|
|
struct kvm_memslots *slots = kvm_memslots(kvm);
|
|
|
|
struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
|
2015-01-16 00:58:56 +01:00
|
|
|
phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
|
|
|
|
phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
|
|
|
|
|
|
|
|
spin_lock(&kvm->mmu_lock);
|
|
|
|
stage2_wp_range(kvm, start, end);
|
|
|
|
spin_unlock(&kvm->mmu_lock);
|
|
|
|
kvm_flush_remote_tlbs(kvm);
|
|
|
|
}
|
2015-01-16 00:58:57 +01:00
|
|
|
|
|
|
|
/**
|
2015-01-28 03:54:23 +01:00
|
|
|
* kvm_mmu_write_protect_pt_masked() - write protect dirty pages
|
2015-01-16 00:58:57 +01:00
|
|
|
* @kvm: The KVM pointer
|
|
|
|
* @slot: The memory slot associated with mask
|
|
|
|
* @gfn_offset: The gfn offset in memory slot
|
|
|
|
* @mask: The mask of dirty pages at offset 'gfn_offset' in this memory
|
|
|
|
* slot to be write protected
|
|
|
|
*
|
|
|
|
* Walks bits set in mask write protects the associated pte's. Caller must
|
|
|
|
* acquire kvm_mmu_lock.
|
|
|
|
*/
|
2015-01-28 03:54:23 +01:00
|
|
|
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
|
2015-01-16 00:58:57 +01:00
|
|
|
struct kvm_memory_slot *slot,
|
|
|
|
gfn_t gfn_offset, unsigned long mask)
|
|
|
|
{
|
|
|
|
phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
|
|
|
|
phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT;
|
|
|
|
phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
|
|
|
|
|
|
|
|
stage2_wp_range(kvm, start, end);
|
|
|
|
}
|
2015-01-16 00:58:56 +01:00
|
|
|
|
2015-01-28 03:54:23 +01:00
|
|
|
/*
|
|
|
|
* kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
|
|
|
|
* dirty pages.
|
|
|
|
*
|
|
|
|
* It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
|
|
|
|
* enable dirty logging for them.
|
|
|
|
*/
|
|
|
|
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
|
|
|
|
struct kvm_memory_slot *slot,
|
|
|
|
gfn_t gfn_offset, unsigned long mask)
|
|
|
|
{
|
|
|
|
kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
|
|
|
|
}
|
|
|
|
|
kvm: rename pfn_t to kvm_pfn_t
To date, we have implemented two I/O usage models for persistent memory,
PMEM (a persistent "ram disk") and DAX (mmap persistent memory into
userspace). This series adds a third, DAX-GUP, that allows DAX mappings
to be the target of direct-i/o. It allows userspace to coordinate
DMA/RDMA from/to persistent memory.
The implementation leverages the ZONE_DEVICE mm-zone that went into
4.3-rc1 (also discussed at kernel summit) to flag pages that are owned
and dynamically mapped by a device driver. The pmem driver, after
mapping a persistent memory range into the system memmap via
devm_memremap_pages(), arranges for DAX to distinguish pfn-only versus
page-backed pmem-pfns via flags in the new pfn_t type.
The DAX code, upon seeing a PFN_DEV+PFN_MAP flagged pfn, flags the
resulting pte(s) inserted into the process page tables with a new
_PAGE_DEVMAP flag. Later, when get_user_pages() is walking ptes it keys
off _PAGE_DEVMAP to pin the device hosting the page range active.
Finally, get_page() and put_page() are modified to take references
against the device driver established page mapping.
Finally, this need for "struct page" for persistent memory requires
memory capacity to store the memmap array. Given the memmap array for a
large pool of persistent may exhaust available DRAM introduce a
mechanism to allocate the memmap from persistent memory. The new
"struct vmem_altmap *" parameter to devm_memremap_pages() enables
arch_add_memory() to use reserved pmem capacity rather than the page
allocator.
This patch (of 18):
The core has developed a need for a "pfn_t" type [1]. Move the existing
pfn_t in KVM to kvm_pfn_t [2].
[1]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002199.html
[2]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002218.html
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-16 01:56:11 +01:00
|
|
|
static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
|
arm/arm64: KVM: Use kernel mapping to perform invalidation on page fault
When handling a fault in stage-2, we need to resync I$ and D$, just
to be sure we don't leave any old cache line behind.
That's very good, except that we do so using the *user* address.
Under heavy load (swapping like crazy), we may end up in a situation
where the page gets mapped in stage-2 while being unmapped from
userspace by another CPU.
At that point, the DC/IC instructions can generate a fault, which
we handle with kvm->mmu_lock held. The box quickly deadlocks, user
is unhappy.
Instead, perform this invalidation through the kernel mapping,
which is guaranteed to be present. The box is much happier, and so
am I.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2015-01-05 22:13:24 +01:00
|
|
|
unsigned long size, bool uncached)
|
|
|
|
{
|
|
|
|
__coherent_cache_guest_page(vcpu, pfn, size, uncached);
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:12 +01:00
|
|
|
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
|
2014-08-19 12:18:04 +02:00
|
|
|
struct kvm_memory_slot *memslot, unsigned long hva,
|
2013-01-21 00:28:12 +01:00
|
|
|
unsigned long fault_status)
|
|
|
|
{
|
|
|
|
int ret;
|
2013-10-03 00:32:01 +02:00
|
|
|
bool write_fault, writable, hugetlb = false, force_pte = false;
|
2013-01-21 00:28:12 +01:00
|
|
|
unsigned long mmu_seq;
|
2012-11-01 17:14:45 +01:00
|
|
|
gfn_t gfn = fault_ipa >> PAGE_SHIFT;
|
|
|
|
struct kvm *kvm = vcpu->kvm;
|
2013-01-21 00:28:12 +01:00
|
|
|
struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
|
2012-11-01 17:14:45 +01:00
|
|
|
struct vm_area_struct *vma;
|
kvm: rename pfn_t to kvm_pfn_t
To date, we have implemented two I/O usage models for persistent memory,
PMEM (a persistent "ram disk") and DAX (mmap persistent memory into
userspace). This series adds a third, DAX-GUP, that allows DAX mappings
to be the target of direct-i/o. It allows userspace to coordinate
DMA/RDMA from/to persistent memory.
The implementation leverages the ZONE_DEVICE mm-zone that went into
4.3-rc1 (also discussed at kernel summit) to flag pages that are owned
and dynamically mapped by a device driver. The pmem driver, after
mapping a persistent memory range into the system memmap via
devm_memremap_pages(), arranges for DAX to distinguish pfn-only versus
page-backed pmem-pfns via flags in the new pfn_t type.
The DAX code, upon seeing a PFN_DEV+PFN_MAP flagged pfn, flags the
resulting pte(s) inserted into the process page tables with a new
_PAGE_DEVMAP flag. Later, when get_user_pages() is walking ptes it keys
off _PAGE_DEVMAP to pin the device hosting the page range active.
Finally, get_page() and put_page() are modified to take references
against the device driver established page mapping.
Finally, this need for "struct page" for persistent memory requires
memory capacity to store the memmap array. Given the memmap array for a
large pool of persistent may exhaust available DRAM introduce a
mechanism to allocate the memmap from persistent memory. The new
"struct vmem_altmap *" parameter to devm_memremap_pages() enables
arch_add_memory() to use reserved pmem capacity rather than the page
allocator.
This patch (of 18):
The core has developed a need for a "pfn_t" type [1]. Move the existing
pfn_t in KVM to kvm_pfn_t [2].
[1]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002199.html
[2]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002218.html
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-16 01:56:11 +01:00
|
|
|
kvm_pfn_t pfn;
|
2014-06-26 02:45:51 +02:00
|
|
|
pgprot_t mem_type = PAGE_S2;
|
2014-11-17 15:58:52 +01:00
|
|
|
bool fault_ipa_uncached;
|
2015-01-16 00:58:58 +01:00
|
|
|
bool logging_active = memslot_is_logging(memslot);
|
|
|
|
unsigned long flags = 0;
|
2013-01-21 00:28:12 +01:00
|
|
|
|
2014-09-09 12:27:09 +02:00
|
|
|
write_fault = kvm_is_write_fault(vcpu);
|
2013-01-21 00:28:12 +01:00
|
|
|
if (fault_status == FSC_PERM && !write_fault) {
|
|
|
|
kvm_err("Unexpected L2 read permission error\n");
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
/* Let's check if we will get back a huge page backed by hugetlbfs */
|
|
|
|
down_read(¤t->mm->mmap_sem);
|
|
|
|
vma = find_vma_intersection(current->mm, hva, hva + 1);
|
2014-09-17 23:56:17 +02:00
|
|
|
if (unlikely(!vma)) {
|
|
|
|
kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
|
|
|
|
up_read(¤t->mm->mmap_sem);
|
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
if (is_vm_hugetlb_page(vma) && !logging_active) {
|
2012-11-01 17:14:45 +01:00
|
|
|
hugetlb = true;
|
|
|
|
gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
|
2013-10-03 00:32:01 +02:00
|
|
|
} else {
|
|
|
|
/*
|
2013-12-13 17:56:06 +01:00
|
|
|
* Pages belonging to memslots that don't have the same
|
|
|
|
* alignment for userspace and IPA cannot be mapped using
|
|
|
|
* block descriptors even if the pages belong to a THP for
|
|
|
|
* the process, because the stage-2 block descriptor will
|
|
|
|
* cover more than a single THP and we loose atomicity for
|
|
|
|
* unmapping, updates, and splits of the THP or other pages
|
|
|
|
* in the stage-2 block range.
|
2013-10-03 00:32:01 +02:00
|
|
|
*/
|
2013-12-13 17:56:06 +01:00
|
|
|
if ((memslot->userspace_addr & ~PMD_MASK) !=
|
|
|
|
((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK))
|
2013-10-03 00:32:01 +02:00
|
|
|
force_pte = true;
|
2012-11-01 17:14:45 +01:00
|
|
|
}
|
|
|
|
up_read(¤t->mm->mmap_sem);
|
|
|
|
|
2013-01-21 00:28:12 +01:00
|
|
|
/* We need minimum second+third level pages */
|
2014-10-10 12:14:28 +02:00
|
|
|
ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES,
|
|
|
|
KVM_NR_MEM_OBJS);
|
2013-01-21 00:28:12 +01:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
mmu_seq = vcpu->kvm->mmu_notifier_seq;
|
|
|
|
/*
|
|
|
|
* Ensure the read of mmu_notifier_seq happens before we call
|
|
|
|
* gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
|
|
|
|
* the page we just got a reference to gets unmapped before we have a
|
|
|
|
* chance to grab the mmu_lock, which ensure that if the page gets
|
|
|
|
* unmapped afterwards, the call to kvm_unmap_hva will take it away
|
|
|
|
* from us again properly. This smp_rmb() interacts with the smp_wmb()
|
|
|
|
* in kvm_mmu_notifier_invalidate_<page|range_end>.
|
|
|
|
*/
|
|
|
|
smp_rmb();
|
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);
|
2016-08-17 10:46:10 +02:00
|
|
|
if (is_error_noslot_pfn(pfn))
|
2013-01-21 00:28:12 +01:00
|
|
|
return -EFAULT;
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
if (kvm_is_device_pfn(pfn)) {
|
2014-06-26 02:45:51 +02:00
|
|
|
mem_type = PAGE_S2_DEVICE;
|
2015-01-16 00:58:58 +01:00
|
|
|
flags |= KVM_S2PTE_FLAG_IS_IOMAP;
|
|
|
|
} else if (logging_active) {
|
|
|
|
/*
|
|
|
|
* Faults on pages in a memslot with logging enabled
|
|
|
|
* should not be mapped with huge pages (it introduces churn
|
|
|
|
* and performance degradation), so force a pte mapping.
|
|
|
|
*/
|
|
|
|
force_pte = true;
|
|
|
|
flags |= KVM_S2_FLAG_LOGGING_ACTIVE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only actually map the page as writable if this was a write
|
|
|
|
* fault.
|
|
|
|
*/
|
|
|
|
if (!write_fault)
|
|
|
|
writable = false;
|
|
|
|
}
|
2014-06-26 02:45:51 +02:00
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
spin_lock(&kvm->mmu_lock);
|
|
|
|
if (mmu_notifier_retry(kvm, mmu_seq))
|
2013-01-21 00:28:12 +01:00
|
|
|
goto out_unlock;
|
2015-01-16 00:58:58 +01:00
|
|
|
|
2013-10-03 00:32:01 +02:00
|
|
|
if (!hugetlb && !force_pte)
|
|
|
|
hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
|
2012-11-01 17:14:45 +01:00
|
|
|
|
2014-11-17 15:58:53 +01:00
|
|
|
fault_ipa_uncached = memslot->flags & KVM_MEMSLOT_INCOHERENT;
|
2014-11-17 15:58:52 +01:00
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
if (hugetlb) {
|
2014-06-26 02:45:51 +02:00
|
|
|
pmd_t new_pmd = pfn_pmd(pfn, mem_type);
|
2012-11-01 17:14:45 +01:00
|
|
|
new_pmd = pmd_mkhuge(new_pmd);
|
|
|
|
if (writable) {
|
2016-04-13 18:57:37 +02:00
|
|
|
new_pmd = kvm_s2pmd_mkwrite(new_pmd);
|
2012-11-01 17:14:45 +01:00
|
|
|
kvm_set_pfn_dirty(pfn);
|
|
|
|
}
|
arm/arm64: KVM: Use kernel mapping to perform invalidation on page fault
When handling a fault in stage-2, we need to resync I$ and D$, just
to be sure we don't leave any old cache line behind.
That's very good, except that we do so using the *user* address.
Under heavy load (swapping like crazy), we may end up in a situation
where the page gets mapped in stage-2 while being unmapped from
userspace by another CPU.
At that point, the DC/IC instructions can generate a fault, which
we handle with kvm->mmu_lock held. The box quickly deadlocks, user
is unhappy.
Instead, perform this invalidation through the kernel mapping,
which is guaranteed to be present. The box is much happier, and so
am I.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2015-01-05 22:13:24 +01:00
|
|
|
coherent_cache_guest_page(vcpu, pfn, PMD_SIZE, fault_ipa_uncached);
|
2012-11-01 17:14:45 +01:00
|
|
|
ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
|
|
|
|
} else {
|
2014-06-26 02:45:51 +02:00
|
|
|
pte_t new_pte = pfn_pte(pfn, mem_type);
|
2015-01-16 00:58:58 +01:00
|
|
|
|
2012-11-01 17:14:45 +01:00
|
|
|
if (writable) {
|
2016-04-13 18:57:37 +02:00
|
|
|
new_pte = kvm_s2pte_mkwrite(new_pte);
|
2012-11-01 17:14:45 +01:00
|
|
|
kvm_set_pfn_dirty(pfn);
|
2015-01-16 00:58:58 +01:00
|
|
|
mark_page_dirty(kvm, gfn);
|
2012-11-01 17:14:45 +01:00
|
|
|
}
|
arm/arm64: KVM: Use kernel mapping to perform invalidation on page fault
When handling a fault in stage-2, we need to resync I$ and D$, just
to be sure we don't leave any old cache line behind.
That's very good, except that we do so using the *user* address.
Under heavy load (swapping like crazy), we may end up in a situation
where the page gets mapped in stage-2 while being unmapped from
userspace by another CPU.
At that point, the DC/IC instructions can generate a fault, which
we handle with kvm->mmu_lock held. The box quickly deadlocks, user
is unhappy.
Instead, perform this invalidation through the kernel mapping,
which is guaranteed to be present. The box is much happier, and so
am I.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2015-01-05 22:13:24 +01:00
|
|
|
coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE, fault_ipa_uncached);
|
2015-01-16 00:58:58 +01:00
|
|
|
ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
|
2013-01-21 00:28:12 +01:00
|
|
|
}
|
2012-11-01 17:14:45 +01:00
|
|
|
|
2013-01-21 00:28:12 +01:00
|
|
|
out_unlock:
|
2012-11-01 17:14:45 +01:00
|
|
|
spin_unlock(&kvm->mmu_lock);
|
2015-03-12 19:16:51 +01:00
|
|
|
kvm_set_pfn_accessed(pfn);
|
2013-01-21 00:28:12 +01:00
|
|
|
kvm_release_pfn_clean(pfn);
|
2012-11-01 17:14:45 +01:00
|
|
|
return ret;
|
2013-01-21 00:28:12 +01:00
|
|
|
}
|
|
|
|
|
2015-03-12 19:16:52 +01:00
|
|
|
/*
|
|
|
|
* Resolve the access fault by making the page young again.
|
|
|
|
* Note that because the faulting entry is guaranteed not to be
|
|
|
|
* cached in the TLB, we don't need to invalidate anything.
|
2016-04-13 18:57:37 +02:00
|
|
|
* Only the HW Access Flag updates are supported for Stage 2 (no DBM),
|
|
|
|
* so there is no need for atomic (pte|pmd)_mkyoung operations.
|
2015-03-12 19:16:52 +01:00
|
|
|
*/
|
|
|
|
static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
|
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
kvm: rename pfn_t to kvm_pfn_t
To date, we have implemented two I/O usage models for persistent memory,
PMEM (a persistent "ram disk") and DAX (mmap persistent memory into
userspace). This series adds a third, DAX-GUP, that allows DAX mappings
to be the target of direct-i/o. It allows userspace to coordinate
DMA/RDMA from/to persistent memory.
The implementation leverages the ZONE_DEVICE mm-zone that went into
4.3-rc1 (also discussed at kernel summit) to flag pages that are owned
and dynamically mapped by a device driver. The pmem driver, after
mapping a persistent memory range into the system memmap via
devm_memremap_pages(), arranges for DAX to distinguish pfn-only versus
page-backed pmem-pfns via flags in the new pfn_t type.
The DAX code, upon seeing a PFN_DEV+PFN_MAP flagged pfn, flags the
resulting pte(s) inserted into the process page tables with a new
_PAGE_DEVMAP flag. Later, when get_user_pages() is walking ptes it keys
off _PAGE_DEVMAP to pin the device hosting the page range active.
Finally, get_page() and put_page() are modified to take references
against the device driver established page mapping.
Finally, this need for "struct page" for persistent memory requires
memory capacity to store the memmap array. Given the memmap array for a
large pool of persistent may exhaust available DRAM introduce a
mechanism to allocate the memmap from persistent memory. The new
"struct vmem_altmap *" parameter to devm_memremap_pages() enables
arch_add_memory() to use reserved pmem capacity rather than the page
allocator.
This patch (of 18):
The core has developed a need for a "pfn_t" type [1]. Move the existing
pfn_t in KVM to kvm_pfn_t [2].
[1]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002199.html
[2]: https://lists.01.org/pipermail/linux-nvdimm/2015-September/002218.html
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-16 01:56:11 +01:00
|
|
|
kvm_pfn_t pfn;
|
2015-03-12 19:16:52 +01:00
|
|
|
bool pfn_valid = false;
|
|
|
|
|
|
|
|
trace_kvm_access_fault(fault_ipa);
|
|
|
|
|
|
|
|
spin_lock(&vcpu->kvm->mmu_lock);
|
|
|
|
|
|
|
|
pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
|
|
|
|
if (!pmd || pmd_none(*pmd)) /* Nothing there */
|
|
|
|
goto out;
|
|
|
|
|
2016-03-01 13:00:39 +01:00
|
|
|
if (pmd_thp_or_huge(*pmd)) { /* THP, HugeTLB */
|
2015-03-12 19:16:52 +01:00
|
|
|
*pmd = pmd_mkyoung(*pmd);
|
|
|
|
pfn = pmd_pfn(*pmd);
|
|
|
|
pfn_valid = true;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
pte = pte_offset_kernel(pmd, fault_ipa);
|
|
|
|
if (pte_none(*pte)) /* Nothing there either */
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
*pte = pte_mkyoung(*pte); /* Just a page... */
|
|
|
|
pfn = pte_pfn(*pte);
|
|
|
|
pfn_valid = true;
|
|
|
|
out:
|
|
|
|
spin_unlock(&vcpu->kvm->mmu_lock);
|
|
|
|
if (pfn_valid)
|
|
|
|
kvm_set_pfn_accessed(pfn);
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:12 +01:00
|
|
|
/**
|
|
|
|
* kvm_handle_guest_abort - handles all 2nd stage aborts
|
|
|
|
* @vcpu: the VCPU pointer
|
|
|
|
* @run: the kvm_run structure
|
|
|
|
*
|
|
|
|
* Any abort that gets to the host is almost guaranteed to be caused by a
|
|
|
|
* missing second stage translation table entry, which can mean that either the
|
|
|
|
* guest simply needs more memory and we must allocate an appropriate page or it
|
|
|
|
* can mean that the guest tried to access I/O memory, which is emulated by user
|
|
|
|
* space. The distinction is based on the IPA causing the fault and whether this
|
|
|
|
* memory region has been registered as standard RAM by user space.
|
|
|
|
*/
|
2013-01-21 00:28:06 +01:00
|
|
|
int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|
|
|
{
|
2013-01-21 00:28:12 +01:00
|
|
|
unsigned long fault_status;
|
|
|
|
phys_addr_t fault_ipa;
|
|
|
|
struct kvm_memory_slot *memslot;
|
2014-08-19 12:18:04 +02:00
|
|
|
unsigned long hva;
|
|
|
|
bool is_iabt, write_fault, writable;
|
2013-01-21 00:28:12 +01:00
|
|
|
gfn_t gfn;
|
|
|
|
int ret, idx;
|
|
|
|
|
2012-10-15 11:33:38 +02:00
|
|
|
is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
|
2016-09-06 15:02:15 +02:00
|
|
|
if (unlikely(!is_iabt && kvm_vcpu_dabt_isextabt(vcpu))) {
|
|
|
|
kvm_inject_vabt(vcpu);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2012-09-17 20:27:09 +02:00
|
|
|
fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
|
2013-01-21 00:28:12 +01:00
|
|
|
|
2012-09-17 20:27:09 +02:00
|
|
|
trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu),
|
|
|
|
kvm_vcpu_get_hfar(vcpu), fault_ipa);
|
2013-01-21 00:28:12 +01:00
|
|
|
|
|
|
|
/* Check the stage-2 fault is trans. fault or write fault */
|
2014-09-26 12:29:34 +02:00
|
|
|
fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
|
2015-03-12 19:16:51 +01:00
|
|
|
if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
|
|
|
|
fault_status != FSC_ACCESS) {
|
2014-09-26 12:29:34 +02:00
|
|
|
kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
|
|
|
|
kvm_vcpu_trap_get_class(vcpu),
|
|
|
|
(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
|
|
|
|
(unsigned long)kvm_vcpu_get_hsr(vcpu));
|
2013-01-21 00:28:12 +01:00
|
|
|
return -EFAULT;
|
|
|
|
}
|
|
|
|
|
|
|
|
idx = srcu_read_lock(&vcpu->kvm->srcu);
|
|
|
|
|
|
|
|
gfn = fault_ipa >> PAGE_SHIFT;
|
2014-08-19 12:18:04 +02:00
|
|
|
memslot = gfn_to_memslot(vcpu->kvm, gfn);
|
|
|
|
hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
|
2014-09-09 12:27:09 +02:00
|
|
|
write_fault = kvm_is_write_fault(vcpu);
|
2014-08-19 12:18:04 +02:00
|
|
|
if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
|
2013-01-21 00:28:12 +01:00
|
|
|
if (is_iabt) {
|
|
|
|
/* Prefetch Abort on I/O address */
|
2012-09-17 20:27:09 +02:00
|
|
|
kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
|
2013-01-21 00:28:12 +01:00
|
|
|
ret = 1;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2016-01-29 16:01:28 +01:00
|
|
|
/*
|
|
|
|
* Check for a cache maintenance operation. Since we
|
|
|
|
* ended-up here, we know it is outside of any memory
|
|
|
|
* slot. But we can't find out if that is for a device,
|
|
|
|
* or if the guest is just being stupid. The only thing
|
|
|
|
* we know for sure is that this range cannot be cached.
|
|
|
|
*
|
|
|
|
* So let's assume that the guest is just being
|
|
|
|
* cautious, and skip the instruction.
|
|
|
|
*/
|
|
|
|
if (kvm_vcpu_dabt_is_cm(vcpu)) {
|
|
|
|
kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
|
|
|
|
ret = 1;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2012-12-12 15:42:09 +01:00
|
|
|
/*
|
|
|
|
* The IPA is reported as [MAX:12], so we need to
|
|
|
|
* complement it with the bottom 12 bits from the
|
|
|
|
* faulting VA. This is always 12 bits, irrespective
|
|
|
|
* of the page size.
|
|
|
|
*/
|
|
|
|
fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
|
2013-01-21 00:43:58 +01:00
|
|
|
ret = io_mem_abort(vcpu, run, fault_ipa);
|
2013-01-21 00:28:12 +01:00
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2014-10-10 12:14:29 +02:00
|
|
|
/* Userspace should not be able to register out-of-bounds IPAs */
|
|
|
|
VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
|
|
|
|
|
2015-03-12 19:16:52 +01:00
|
|
|
if (fault_status == FSC_ACCESS) {
|
|
|
|
handle_access_fault(vcpu, fault_ipa);
|
|
|
|
ret = 1;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
|
2014-08-19 12:18:04 +02:00
|
|
|
ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
|
2013-01-21 00:28:12 +01:00
|
|
|
if (ret == 0)
|
|
|
|
ret = 1;
|
|
|
|
out_unlock:
|
|
|
|
srcu_read_unlock(&vcpu->kvm->srcu, idx);
|
|
|
|
return ret;
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
2015-03-12 19:16:50 +01:00
|
|
|
static int handle_hva_to_gpa(struct kvm *kvm,
|
|
|
|
unsigned long start,
|
|
|
|
unsigned long end,
|
|
|
|
int (*handler)(struct kvm *kvm,
|
|
|
|
gpa_t gpa, void *data),
|
|
|
|
void *data)
|
2013-01-21 00:28:07 +01:00
|
|
|
{
|
|
|
|
struct kvm_memslots *slots;
|
|
|
|
struct kvm_memory_slot *memslot;
|
2015-03-12 19:16:50 +01:00
|
|
|
int ret = 0;
|
2013-01-21 00:28:07 +01:00
|
|
|
|
|
|
|
slots = kvm_memslots(kvm);
|
|
|
|
|
|
|
|
/* we only care about the pages that the guest sees */
|
|
|
|
kvm_for_each_memslot(memslot, slots) {
|
|
|
|
unsigned long hva_start, hva_end;
|
|
|
|
gfn_t gfn, gfn_end;
|
|
|
|
|
|
|
|
hva_start = max(start, memslot->userspace_addr);
|
|
|
|
hva_end = min(end, memslot->userspace_addr +
|
|
|
|
(memslot->npages << PAGE_SHIFT));
|
|
|
|
if (hva_start >= hva_end)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* {gfn(page) | page intersects with [hva_start, hva_end)} =
|
|
|
|
* {gfn_start, gfn_start+1, ..., gfn_end-1}.
|
|
|
|
*/
|
|
|
|
gfn = hva_to_gfn_memslot(hva_start, memslot);
|
|
|
|
gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
|
|
|
|
|
|
|
|
for (; gfn < gfn_end; ++gfn) {
|
|
|
|
gpa_t gpa = gfn << PAGE_SHIFT;
|
2015-03-12 19:16:50 +01:00
|
|
|
ret |= handler(kvm, gpa, data);
|
2013-01-21 00:28:07 +01:00
|
|
|
}
|
|
|
|
}
|
2015-03-12 19:16:50 +01:00
|
|
|
|
|
|
|
return ret;
|
2013-01-21 00:28:07 +01:00
|
|
|
}
|
|
|
|
|
2015-03-12 19:16:50 +01:00
|
|
|
static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
|
2013-01-21 00:28:07 +01:00
|
|
|
{
|
|
|
|
unmap_stage2_range(kvm, gpa, PAGE_SIZE);
|
2015-03-12 19:16:50 +01:00
|
|
|
return 0;
|
2013-01-21 00:28:07 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
|
|
|
|
{
|
|
|
|
unsigned long end = hva + PAGE_SIZE;
|
|
|
|
|
|
|
|
if (!kvm->arch.pgd)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
trace_kvm_unmap_hva(hva);
|
|
|
|
handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_unmap_hva_range(struct kvm *kvm,
|
|
|
|
unsigned long start, unsigned long end)
|
|
|
|
{
|
|
|
|
if (!kvm->arch.pgd)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
trace_kvm_unmap_hva_range(start, end);
|
|
|
|
handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-03-12 19:16:50 +01:00
|
|
|
static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
|
2013-01-21 00:28:07 +01:00
|
|
|
{
|
|
|
|
pte_t *pte = (pte_t *)data;
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
/*
|
|
|
|
* We can always call stage2_set_pte with KVM_S2PTE_FLAG_LOGGING_ACTIVE
|
|
|
|
* flag clear because MMU notifiers will have unmapped a huge PMD before
|
|
|
|
* calling ->change_pte() (which in turn calls kvm_set_spte_hva()) and
|
|
|
|
* therefore stage2_set_pte() never needs to clear out a huge PMD
|
|
|
|
* through this calling path.
|
|
|
|
*/
|
|
|
|
stage2_set_pte(kvm, NULL, gpa, pte, 0);
|
2015-03-12 19:16:50 +01:00
|
|
|
return 0;
|
2013-01-21 00:28:07 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
|
|
|
|
{
|
|
|
|
unsigned long end = hva + PAGE_SIZE;
|
|
|
|
pte_t stage2_pte;
|
|
|
|
|
|
|
|
if (!kvm->arch.pgd)
|
|
|
|
return;
|
|
|
|
|
|
|
|
trace_kvm_set_spte_hva(hva);
|
|
|
|
stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2);
|
|
|
|
handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
|
|
|
|
}
|
|
|
|
|
2015-03-12 19:16:51 +01:00
|
|
|
static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
|
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
|
|
|
|
|
|
|
pmd = stage2_get_pmd(kvm, NULL, gpa);
|
|
|
|
if (!pmd || pmd_none(*pmd)) /* Nothing there */
|
|
|
|
return 0;
|
|
|
|
|
2016-04-13 18:57:37 +02:00
|
|
|
if (pmd_thp_or_huge(*pmd)) /* THP, HugeTLB */
|
|
|
|
return stage2_pmdp_test_and_clear_young(pmd);
|
2015-03-12 19:16:51 +01:00
|
|
|
|
|
|
|
pte = pte_offset_kernel(pmd, gpa);
|
|
|
|
if (pte_none(*pte))
|
|
|
|
return 0;
|
|
|
|
|
2016-04-13 18:57:37 +02:00
|
|
|
return stage2_ptep_test_and_clear_young(pte);
|
2015-03-12 19:16:51 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
|
|
|
|
{
|
|
|
|
pmd_t *pmd;
|
|
|
|
pte_t *pte;
|
|
|
|
|
|
|
|
pmd = stage2_get_pmd(kvm, NULL, gpa);
|
|
|
|
if (!pmd || pmd_none(*pmd)) /* Nothing there */
|
|
|
|
return 0;
|
|
|
|
|
2016-03-01 13:00:39 +01:00
|
|
|
if (pmd_thp_or_huge(*pmd)) /* THP, HugeTLB */
|
2015-03-12 19:16:51 +01:00
|
|
|
return pmd_young(*pmd);
|
|
|
|
|
|
|
|
pte = pte_offset_kernel(pmd, gpa);
|
|
|
|
if (!pte_none(*pte)) /* Just a page... */
|
|
|
|
return pte_young(*pte);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
|
|
|
|
{
|
|
|
|
trace_kvm_age_hva(start, end);
|
|
|
|
return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
|
|
|
|
{
|
|
|
|
trace_kvm_test_age_hva(hva);
|
|
|
|
return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:07 +01:00
|
|
|
void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:06 +01:00
|
|
|
phys_addr_t kvm_mmu_get_httbr(void)
|
|
|
|
{
|
2015-03-19 17:42:28 +01:00
|
|
|
if (__kvm_cpu_uses_extended_idmap())
|
|
|
|
return virt_to_phys(merged_hyp_pgd);
|
|
|
|
else
|
|
|
|
return virt_to_phys(hyp_pgd);
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
|
|
|
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
phys_addr_t kvm_get_idmap_vector(void)
|
|
|
|
{
|
|
|
|
return hyp_idmap_vector;
|
|
|
|
}
|
|
|
|
|
arm64: kvm: allows kvm cpu hotplug
The current kvm implementation on arm64 does cpu-specific initialization
at system boot, and has no way to gracefully shutdown a core in terms of
kvm. This prevents kexec from rebooting the system at EL2.
This patch adds a cpu tear-down function and also puts an existing cpu-init
code into a separate function, kvm_arch_hardware_disable() and
kvm_arch_hardware_enable() respectively.
We don't need the arm64 specific cpu hotplug hook any more.
Since this patch modifies common code between arm and arm64, one stub
definition, __cpu_reset_hyp_mode(), is added on arm side to avoid
compilation errors.
Signed-off-by: AKASHI Takahiro <takahiro.akashi@linaro.org>
[Rebase, added separate VHE init/exit path, changed resets use of
kvm_call_hyp() to the __version, en/disabled hardware in init_subsystems(),
added icache maintenance to __kvm_hyp_reset() and removed lr restore, removed
guest-enter after teardown handling]
Signed-off-by: James Morse <james.morse@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-04-27 18:47:05 +02:00
|
|
|
phys_addr_t kvm_get_idmap_start(void)
|
|
|
|
{
|
|
|
|
return hyp_idmap_start;
|
|
|
|
}
|
|
|
|
|
2016-06-30 19:40:43 +02:00
|
|
|
static int kvm_map_idmap_text(pgd_t *pgd)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
/* Create the idmap in the boot page tables */
|
|
|
|
err = __create_hyp_mappings(pgd,
|
|
|
|
hyp_idmap_start, hyp_idmap_end,
|
|
|
|
__phys_to_pfn(hyp_idmap_start),
|
|
|
|
PAGE_HYP_EXEC);
|
|
|
|
if (err)
|
|
|
|
kvm_err("Failed to idmap %lx-%lx\n",
|
|
|
|
hyp_idmap_start, hyp_idmap_end);
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:06 +01:00
|
|
|
int kvm_mmu_init(void)
|
|
|
|
{
|
2013-04-12 20:12:03 +02:00
|
|
|
int err;
|
|
|
|
|
2013-11-19 20:59:12 +01:00
|
|
|
hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start);
|
|
|
|
hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end);
|
|
|
|
hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init);
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
|
ARM, arm64: kvm: get rid of the bounce page
The HYP init bounce page is a runtime construct that ensures that the
HYP init code does not cross a page boundary. However, this is something
we can do perfectly well at build time, by aligning the code appropriately.
For arm64, we just align to 4 KB, and enforce that the code size is less
than 4 KB, regardless of the chosen page size.
For ARM, the whole code is less than 256 bytes, so we tweak the linker
script to align at a power of 2 upper bound of the code size
Note that this also fixes a benign off-by-one error in the original bounce
page code, where a bounce page would be allocated unnecessarily if the code
was exactly 1 page in size.
On ARM, it also fixes an issue with very large kernels reported by Arnd
Bergmann, where stub sections with linker emitted veneers could erroneously
trigger the size/alignment ASSERT() in the linker script.
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2015-03-19 17:42:26 +01:00
|
|
|
/*
|
|
|
|
* We rely on the linker script to ensure at build time that the HYP
|
|
|
|
* init code does not cross a page boundary.
|
|
|
|
*/
|
|
|
|
BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
|
2016-06-30 19:40:50 +02:00
|
|
|
kvm_info("IDMAP page: %lx\n", hyp_idmap_start);
|
|
|
|
kvm_info("HYP VA range: %lx:%lx\n",
|
2016-06-30 19:40:51 +02:00
|
|
|
kern_hyp_va(PAGE_OFFSET), kern_hyp_va(~0UL));
|
2016-06-30 19:40:50 +02:00
|
|
|
|
2016-06-30 19:40:51 +02:00
|
|
|
if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
|
2016-08-22 10:01:17 +02:00
|
|
|
hyp_idmap_start < kern_hyp_va(~0UL) &&
|
|
|
|
hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) {
|
2016-06-30 19:40:50 +02:00
|
|
|
/*
|
|
|
|
* The idmap page is intersecting with the VA space,
|
|
|
|
* it is not safe to continue further.
|
|
|
|
*/
|
|
|
|
kvm_err("IDMAP intersecting with HYP VA, unable to continue\n");
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2014-10-10 12:14:28 +02:00
|
|
|
hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order);
|
2016-06-30 19:40:43 +02:00
|
|
|
if (!hyp_pgd) {
|
2013-01-21 00:28:07 +01:00
|
|
|
kvm_err("Hyp mode PGD not allocated\n");
|
2013-04-12 20:12:03 +02:00
|
|
|
err = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2016-06-30 19:40:43 +02:00
|
|
|
if (__kvm_cpu_uses_extended_idmap()) {
|
|
|
|
boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
|
|
|
|
hyp_pgd_order);
|
|
|
|
if (!boot_hyp_pgd) {
|
|
|
|
kvm_err("Hyp boot PGD not allocated\n");
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
2013-04-12 20:12:03 +02:00
|
|
|
|
2016-06-30 19:40:43 +02:00
|
|
|
err = kvm_map_idmap_text(boot_hyp_pgd);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
2013-01-21 00:28:07 +01:00
|
|
|
|
2015-03-19 17:42:28 +01:00
|
|
|
merged_hyp_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
|
|
|
|
if (!merged_hyp_pgd) {
|
|
|
|
kvm_err("Failed to allocate extra HYP pgd\n");
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
__kvm_extend_hypmap(boot_hyp_pgd, hyp_pgd, merged_hyp_pgd,
|
|
|
|
hyp_idmap_start);
|
2016-06-30 19:40:43 +02:00
|
|
|
} else {
|
|
|
|
err = kvm_map_idmap_text(hyp_pgd);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
ARM: KVM: switch to a dual-step HYP init code
Our HYP init code suffers from two major design issues:
- it cannot support CPU hotplug, as we tear down the idmap very early
- it cannot perform a TLB invalidation when switching from init to
runtime mappings, as pages are manipulated from PL1 exclusively
The hotplug problem mandates that we keep two sets of page tables
(boot and runtime). The TLB problem mandates that we're able to
transition from one PGD to another while in HYP, invalidating the TLBs
in the process.
To be able to do this, we need to share a page between the two page
tables. A page that will have the same VA in both configurations. All we
need is a VA that has the following properties:
- This VA can't be used to represent a kernel mapping.
- This VA will not conflict with the physical address of the kernel text
The vectors page seems to satisfy this requirement:
- The kernel never maps anything else there
- The kernel text being copied at the beginning of the physical memory,
it is unlikely to use the last 64kB (I doubt we'll ever support KVM
on a system with something like 4MB of RAM, but patches are very
welcome).
Let's call this VA the trampoline VA.
Now, we map our init page at 3 locations:
- idmap in the boot pgd
- trampoline VA in the boot pgd
- trampoline VA in the runtime pgd
The init scenario is now the following:
- We jump in HYP with four parameters: boot HYP pgd, runtime HYP pgd,
runtime stack, runtime vectors
- Enable the MMU with the boot pgd
- Jump to a target into the trampoline page (remember, this is the same
physical page!)
- Now switch to the runtime pgd (same VA, and still the same physical
page!)
- Invalidate TLBs
- Set stack and vectors
- Profit! (or eret, if you only care about the code).
Note that we keep the boot mapping permanently (it is not strictly an
idmap anymore) to allow for CPU hotplug in later patches.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <cdall@cs.columbia.edu>
2013-04-12 20:12:06 +02:00
|
|
|
}
|
|
|
|
|
2013-01-21 00:28:07 +01:00
|
|
|
return 0;
|
2013-04-12 20:12:03 +02:00
|
|
|
out:
|
2013-04-12 20:12:05 +02:00
|
|
|
free_hyp_pgds();
|
2013-04-12 20:12:03 +02:00
|
|
|
return err;
|
2013-01-21 00:28:06 +01:00
|
|
|
}
|
2014-06-06 11:10:23 +02:00
|
|
|
|
|
|
|
void kvm_arch_commit_memory_region(struct kvm *kvm,
|
2015-05-18 13:59:39 +02:00
|
|
|
const struct kvm_userspace_memory_region *mem,
|
2014-06-06 11:10:23 +02:00
|
|
|
const struct kvm_memory_slot *old,
|
2015-05-18 13:20:23 +02:00
|
|
|
const struct kvm_memory_slot *new,
|
2014-06-06 11:10:23 +02:00
|
|
|
enum kvm_mr_change change)
|
|
|
|
{
|
2015-01-16 00:58:56 +01:00
|
|
|
/*
|
|
|
|
* At this point memslot has been committed and there is an
|
|
|
|
* allocated dirty_bitmap[], dirty pages will be be tracked while the
|
|
|
|
* memory slot is write protected.
|
|
|
|
*/
|
|
|
|
if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES)
|
|
|
|
kvm_mmu_wp_memory_region(kvm, mem->slot);
|
2014-06-06 11:10:23 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_arch_prepare_memory_region(struct kvm *kvm,
|
|
|
|
struct kvm_memory_slot *memslot,
|
2015-05-18 13:59:39 +02:00
|
|
|
const struct kvm_userspace_memory_region *mem,
|
2014-06-06 11:10:23 +02:00
|
|
|
enum kvm_mr_change change)
|
|
|
|
{
|
2014-10-10 17:00:32 +02:00
|
|
|
hva_t hva = mem->userspace_addr;
|
|
|
|
hva_t reg_end = hva + mem->memory_size;
|
|
|
|
bool writable = !(mem->flags & KVM_MEM_READONLY);
|
|
|
|
int ret = 0;
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
|
|
|
|
change != KVM_MR_FLAGS_ONLY)
|
2014-10-10 17:00:32 +02:00
|
|
|
return 0;
|
|
|
|
|
2014-10-10 12:14:29 +02:00
|
|
|
/*
|
|
|
|
* Prevent userspace from creating a memory region outside of the IPA
|
|
|
|
* space addressable by the KVM guest IPA space.
|
|
|
|
*/
|
|
|
|
if (memslot->base_gfn + memslot->npages >=
|
|
|
|
(KVM_PHYS_SIZE >> PAGE_SHIFT))
|
|
|
|
return -EFAULT;
|
|
|
|
|
2014-10-10 17:00:32 +02:00
|
|
|
/*
|
|
|
|
* A memory region could potentially cover multiple VMAs, and any holes
|
|
|
|
* between them, so iterate over all of them to find out if we can map
|
|
|
|
* any of them right now.
|
|
|
|
*
|
|
|
|
* +--------------------------------------------+
|
|
|
|
* +---------------+----------------+ +----------------+
|
|
|
|
* | : VMA 1 | VMA 2 | | VMA 3 : |
|
|
|
|
* +---------------+----------------+ +----------------+
|
|
|
|
* | memory region |
|
|
|
|
* +--------------------------------------------+
|
|
|
|
*/
|
|
|
|
do {
|
|
|
|
struct vm_area_struct *vma = find_vma(current->mm, hva);
|
|
|
|
hva_t vm_start, vm_end;
|
|
|
|
|
|
|
|
if (!vma || vma->vm_start >= reg_end)
|
|
|
|
break;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Mapping a read-only VMA is only allowed if the
|
|
|
|
* memory region is configured as read-only.
|
|
|
|
*/
|
|
|
|
if (writable && !(vma->vm_flags & VM_WRITE)) {
|
|
|
|
ret = -EPERM;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Take the intersection of this VMA with the memory region
|
|
|
|
*/
|
|
|
|
vm_start = max(hva, vma->vm_start);
|
|
|
|
vm_end = min(reg_end, vma->vm_end);
|
|
|
|
|
|
|
|
if (vma->vm_flags & VM_PFNMAP) {
|
|
|
|
gpa_t gpa = mem->guest_phys_addr +
|
|
|
|
(vm_start - mem->userspace_addr);
|
2015-09-16 12:04:55 +02:00
|
|
|
phys_addr_t pa;
|
|
|
|
|
|
|
|
pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
|
|
|
|
pa += vm_start - vma->vm_start;
|
2014-10-10 17:00:32 +02:00
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
/* IO region dirty page logging not allowed */
|
|
|
|
if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2014-10-10 17:00:32 +02:00
|
|
|
ret = kvm_phys_addr_ioremap(kvm, gpa, pa,
|
|
|
|
vm_end - vm_start,
|
|
|
|
writable);
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
hva = vm_end;
|
|
|
|
} while (hva < reg_end);
|
|
|
|
|
2015-01-16 00:58:58 +01:00
|
|
|
if (change == KVM_MR_FLAGS_ONLY)
|
|
|
|
return ret;
|
|
|
|
|
2014-11-17 15:58:53 +01:00
|
|
|
spin_lock(&kvm->mmu_lock);
|
|
|
|
if (ret)
|
2014-10-10 17:00:32 +02:00
|
|
|
unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
|
2014-11-17 15:58:53 +01:00
|
|
|
else
|
|
|
|
stage2_flush_memslot(kvm, memslot);
|
|
|
|
spin_unlock(&kvm->mmu_lock);
|
2014-10-10 17:00:32 +02:00
|
|
|
return ret;
|
2014-06-06 11:10:23 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
|
|
|
|
struct kvm_memory_slot *dont)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
|
|
|
|
unsigned long npages)
|
|
|
|
{
|
2014-11-17 15:58:53 +01:00
|
|
|
/*
|
|
|
|
* Readonly memslots are not incoherent with the caches by definition,
|
|
|
|
* but in practice, they are used mostly to emulate ROMs or NOR flashes
|
|
|
|
* that the guest may consider devices and hence map as uncached.
|
|
|
|
* To prevent incoherency issues in these cases, tag all readonly
|
|
|
|
* regions as incoherent.
|
|
|
|
*/
|
|
|
|
if (slot->flags & KVM_MEM_READONLY)
|
|
|
|
slot->flags |= KVM_MEMSLOT_INCOHERENT;
|
2014-06-06 11:10:23 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-05-17 21:26:08 +02:00
|
|
|
void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots)
|
2014-06-06 11:10:23 +02:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
void kvm_arch_flush_shadow_all(struct kvm *kvm)
|
|
|
|
{
|
2016-09-08 17:25:49 +02:00
|
|
|
kvm_free_stage2_pgd(kvm);
|
2014-06-06 11:10:23 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
|
|
|
|
struct kvm_memory_slot *slot)
|
|
|
|
{
|
2014-10-10 17:00:32 +02:00
|
|
|
gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
|
|
|
|
phys_addr_t size = slot->npages << PAGE_SHIFT;
|
|
|
|
|
|
|
|
spin_lock(&kvm->mmu_lock);
|
|
|
|
unmap_stage2_range(kvm, gpa, size);
|
|
|
|
spin_unlock(&kvm->mmu_lock);
|
2014-06-06 11:10:23 +02:00
|
|
|
}
|
2014-12-19 17:05:31 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
|
|
|
|
*
|
|
|
|
* Main problems:
|
|
|
|
* - S/W ops are local to a CPU (not broadcast)
|
|
|
|
* - We have line migration behind our back (speculation)
|
|
|
|
* - System caches don't support S/W at all (damn!)
|
|
|
|
*
|
|
|
|
* In the face of the above, the best we can do is to try and convert
|
|
|
|
* S/W ops to VA ops. Because the guest is not allowed to infer the
|
|
|
|
* S/W to PA mapping, it can only use S/W to nuke the whole cache,
|
|
|
|
* which is a rather good thing for us.
|
|
|
|
*
|
|
|
|
* Also, it is only used when turning caches on/off ("The expected
|
|
|
|
* usage of the cache maintenance instructions that operate by set/way
|
|
|
|
* is associated with the cache maintenance instructions associated
|
|
|
|
* with the powerdown and powerup of caches, if this is required by
|
|
|
|
* the implementation.").
|
|
|
|
*
|
|
|
|
* We use the following policy:
|
|
|
|
*
|
|
|
|
* - If we trap a S/W operation, we enable VM trapping to detect
|
|
|
|
* caches being turned on/off, and do a full clean.
|
|
|
|
*
|
|
|
|
* - We flush the caches on both caches being turned on and off.
|
|
|
|
*
|
|
|
|
* - Once the caches are enabled, we stop trapping VM ops.
|
|
|
|
*/
|
|
|
|
void kvm_set_way_flush(struct kvm_vcpu *vcpu)
|
|
|
|
{
|
|
|
|
unsigned long hcr = vcpu_get_hcr(vcpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this is the first time we do a S/W operation
|
|
|
|
* (i.e. HCR_TVM not set) flush the whole memory, and set the
|
|
|
|
* VM trapping.
|
|
|
|
*
|
|
|
|
* Otherwise, rely on the VM trapping to wait for the MMU +
|
|
|
|
* Caches to be turned off. At that point, we'll be able to
|
|
|
|
* clean the caches again.
|
|
|
|
*/
|
|
|
|
if (!(hcr & HCR_TVM)) {
|
|
|
|
trace_kvm_set_way_flush(*vcpu_pc(vcpu),
|
|
|
|
vcpu_has_cache_enabled(vcpu));
|
|
|
|
stage2_flush_vm(vcpu->kvm);
|
|
|
|
vcpu_set_hcr(vcpu, hcr | HCR_TVM);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
|
|
|
|
{
|
|
|
|
bool now_enabled = vcpu_has_cache_enabled(vcpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If switching the MMU+caches on, need to invalidate the caches.
|
|
|
|
* If switching it off, need to clean the caches.
|
|
|
|
* Clean + invalidate does the trick always.
|
|
|
|
*/
|
|
|
|
if (now_enabled != was_enabled)
|
|
|
|
stage2_flush_vm(vcpu->kvm);
|
|
|
|
|
|
|
|
/* Caches are now on, stop trapping VM ops (until a S/W op) */
|
|
|
|
if (now_enabled)
|
|
|
|
vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) & ~HCR_TVM);
|
|
|
|
|
|
|
|
trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
|
|
|
|
}
|