2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* native hashtable management.
|
|
|
|
*
|
|
|
|
* SMP scalability work:
|
|
|
|
* Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*/
|
2005-11-07 01:06:55 +01:00
|
|
|
|
|
|
|
#undef DEBUG_LOW
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/bitops.h>
|
2012-07-25 23:20:03 +02:00
|
|
|
#include <linux/of.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/threads.h>
|
|
|
|
#include <linux/smp.h>
|
|
|
|
|
|
|
|
#include <asm/machdep.h>
|
|
|
|
#include <asm/mmu.h>
|
|
|
|
#include <asm/mmu_context.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/tlb.h>
|
|
|
|
#include <asm/cputable.h>
|
2005-11-07 01:06:55 +01:00
|
|
|
#include <asm/udbg.h>
|
2007-05-02 16:19:11 +02:00
|
|
|
#include <asm/kexec.h>
|
2009-04-29 22:58:01 +02:00
|
|
|
#include <asm/ppc-opcode.h>
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2015-05-27 08:07:16 +02:00
|
|
|
#include <misc/cxl-base.h>
|
2014-10-08 10:55:00 +02:00
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
#ifdef DEBUG_LOW
|
|
|
|
#define DBG_LOW(fmt...) udbg_printf(fmt)
|
|
|
|
#else
|
|
|
|
#define DBG_LOW(fmt...)
|
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2013-09-23 04:04:36 +02:00
|
|
|
#ifdef __BIG_ENDIAN__
|
2005-04-17 00:20:36 +02:00
|
|
|
#define HPTE_LOCK_BIT 3
|
2013-09-23 04:04:36 +02:00
|
|
|
#else
|
|
|
|
#define HPTE_LOCK_BIT (56+3)
|
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
|
KVM: PPC: book3s_hv: Add support for PPC970-family processors
This adds support for running KVM guests in supervisor mode on those
PPC970 processors that have a usable hypervisor mode. Unfortunately,
Apple G5 machines have supervisor mode disabled (MSR[HV] is forced to
1), but the YDL PowerStation does have a usable hypervisor mode.
There are several differences between the PPC970 and POWER7 in how
guests are managed. These differences are accommodated using the
CPU_FTR_ARCH_201 (PPC970) and CPU_FTR_ARCH_206 (POWER7) CPU feature
bits. Notably, on PPC970:
* The LPCR, LPID or RMOR registers don't exist, and the functions of
those registers are provided by bits in HID4 and one bit in HID0.
* External interrupts can be directed to the hypervisor, but unlike
POWER7 they are masked by MSR[EE] in non-hypervisor modes and use
SRR0/1 not HSRR0/1.
* There is no virtual RMA (VRMA) mode; the guest must use an RMO
(real mode offset) area.
* The TLB entries are not tagged with the LPID, so it is necessary to
flush the whole TLB on partition switch. Furthermore, when switching
partitions we have to ensure that no other CPU is executing the tlbie
or tlbsync instructions in either the old or the new partition,
otherwise undefined behaviour can occur.
* The PMU has 8 counters (PMC registers) rather than 6.
* The DSCR, PURR, SPURR, AMR, AMOR, UAMOR registers don't exist.
* The SLB has 64 entries rather than 32.
* There is no mediated external interrupt facility, so if we switch to
a guest that has a virtual external interrupt pending but the guest
has MSR[EE] = 0, we have to arrange to have an interrupt pending for
it so that we can get control back once it re-enables interrupts. We
do that by sending ourselves an IPI with smp_send_reschedule after
hard-disabling interrupts.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 02:40:08 +02:00
|
|
|
DEFINE_RAW_SPINLOCK(native_tlbie_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2013-04-28 11:37:35 +02:00
|
|
|
static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
|
2005-11-07 01:06:55 +01:00
|
|
|
{
|
2012-09-10 04:52:50 +02:00
|
|
|
unsigned long va;
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned int penc;
|
2013-07-04 07:04:45 +02:00
|
|
|
unsigned long sllp;
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
/*
|
|
|
|
* We need 14 to 65 bits of va for a tlibe of 4K page
|
|
|
|
* With vpn we ignore the lower VPN_SHIFT bits already.
|
|
|
|
* And top two bits are already ignored because we can
|
|
|
|
* only accomadate 76 bits in a 64 bit vpn with a VPN_SHIFT
|
|
|
|
* of 12.
|
|
|
|
*/
|
|
|
|
va = vpn << VPN_SHIFT;
|
|
|
|
/*
|
|
|
|
* clear top 16 bits of 64bit va, non SLS segment
|
|
|
|
* Older versions of the architecture (2.02 and earler) require the
|
|
|
|
* masking of the top 16 bits.
|
|
|
|
*/
|
2005-11-07 01:06:55 +01:00
|
|
|
va &= ~(0xffffULL << 48);
|
|
|
|
|
|
|
|
switch (psize) {
|
|
|
|
case MMU_PAGE_4K:
|
2013-04-28 11:37:39 +02:00
|
|
|
/* clear out bits after (52) [0....52.....63] */
|
|
|
|
va &= ~((1ul << (64 - 52)) - 1);
|
2007-10-11 12:37:10 +02:00
|
|
|
va |= ssize << 8;
|
2013-07-04 07:04:45 +02:00
|
|
|
sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) |
|
|
|
|
((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4);
|
|
|
|
va |= sllp << 5;
|
2011-04-06 20:23:29 +02:00
|
|
|
asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
|
powerpc, KVM: Split HVMODE_206 cpu feature bit into separate HV and architecture bits
This replaces the single CPU_FTR_HVMODE_206 bit with two bits, one to
indicate that we have a usable hypervisor mode, and another to indicate
that the processor conforms to PowerISA version 2.06. We also add
another bit to indicate that the processor conforms to ISA version 2.01
and set that for PPC970 and derivatives.
Some PPC970 chips (specifically those in Apple machines) have a
hypervisor mode in that MSR[HV] is always 1, but the hypervisor mode
is not useful in the sense that there is no way to run any code in
supervisor mode (HV=0 PR=0). On these processors, the LPES0 and LPES1
bits in HID4 are always 0, and we use that as a way of detecting that
hypervisor mode is not useful.
Where we have a feature section in assembly code around code that
only applies on POWER7 in hypervisor mode, we use a construct like
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
The definition of END_FTR_SECTION_IFSET is such that the code will
be enabled (not overwritten with nops) only if all bits in the
provided mask are set.
Note that the CPU feature check in __tlbie() only needs to check the
ARCH_206 bit, not the HVMODE bit, because __tlbie() can only get called
if we are running bare-metal, i.e. in hypervisor mode.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 02:26:11 +02:00
|
|
|
: : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
|
2009-04-29 22:58:01 +02:00
|
|
|
: "memory");
|
2005-11-07 01:06:55 +01:00
|
|
|
break;
|
|
|
|
default:
|
2012-09-10 04:52:50 +02:00
|
|
|
/* We need 14 to 14 + i bits of va */
|
2013-04-28 11:37:35 +02:00
|
|
|
penc = mmu_psize_defs[psize].penc[apsize];
|
2013-04-28 11:37:39 +02:00
|
|
|
va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
|
2006-06-15 13:15:44 +02:00
|
|
|
va |= penc << 12;
|
2007-10-11 12:37:10 +02:00
|
|
|
va |= ssize << 8;
|
2014-04-21 07:07:36 +02:00
|
|
|
/*
|
|
|
|
* AVAL bits:
|
|
|
|
* We don't need all the bits, but rest of the bits
|
|
|
|
* must be ignored by the processor.
|
|
|
|
* vpn cover upto 65 bits of va. (0...65) and we need
|
|
|
|
* 58..64 bits of va.
|
|
|
|
*/
|
|
|
|
va |= (vpn & 0xfe); /* AVAL */
|
2009-04-29 22:58:01 +02:00
|
|
|
va |= 1; /* L */
|
2011-04-06 20:23:29 +02:00
|
|
|
asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
|
powerpc, KVM: Split HVMODE_206 cpu feature bit into separate HV and architecture bits
This replaces the single CPU_FTR_HVMODE_206 bit with two bits, one to
indicate that we have a usable hypervisor mode, and another to indicate
that the processor conforms to PowerISA version 2.06. We also add
another bit to indicate that the processor conforms to ISA version 2.01
and set that for PPC970 and derivatives.
Some PPC970 chips (specifically those in Apple machines) have a
hypervisor mode in that MSR[HV] is always 1, but the hypervisor mode
is not useful in the sense that there is no way to run any code in
supervisor mode (HV=0 PR=0). On these processors, the LPES0 and LPES1
bits in HID4 are always 0, and we use that as a way of detecting that
hypervisor mode is not useful.
Where we have a feature section in assembly code around code that
only applies on POWER7 in hypervisor mode, we use a construct like
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
The definition of END_FTR_SECTION_IFSET is such that the code will
be enabled (not overwritten with nops) only if all bits in the
provided mask are set.
Note that the CPU feature check in __tlbie() only needs to check the
ARCH_206 bit, not the HVMODE bit, because __tlbie() can only get called
if we are running bare-metal, i.e. in hypervisor mode.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 02:26:11 +02:00
|
|
|
: : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
|
2009-04-29 22:58:01 +02:00
|
|
|
: "memory");
|
2005-11-07 01:06:55 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-04-28 11:37:35 +02:00
|
|
|
static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
|
2005-11-07 01:06:55 +01:00
|
|
|
{
|
2012-09-10 04:52:50 +02:00
|
|
|
unsigned long va;
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned int penc;
|
2013-07-04 07:04:45 +02:00
|
|
|
unsigned long sllp;
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
/* VPN_SHIFT can be atmost 12 */
|
|
|
|
va = vpn << VPN_SHIFT;
|
|
|
|
/*
|
|
|
|
* clear top 16 bits of 64 bit va, non SLS segment
|
|
|
|
* Older versions of the architecture (2.02 and earler) require the
|
|
|
|
* masking of the top 16 bits.
|
|
|
|
*/
|
2005-11-07 01:06:55 +01:00
|
|
|
va &= ~(0xffffULL << 48);
|
|
|
|
|
|
|
|
switch (psize) {
|
|
|
|
case MMU_PAGE_4K:
|
2013-04-28 11:37:39 +02:00
|
|
|
/* clear out bits after(52) [0....52.....63] */
|
|
|
|
va &= ~((1ul << (64 - 52)) - 1);
|
2007-10-11 12:37:10 +02:00
|
|
|
va |= ssize << 8;
|
2013-07-04 07:04:45 +02:00
|
|
|
sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) |
|
|
|
|
((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4);
|
|
|
|
va |= sllp << 5;
|
2005-11-07 01:06:55 +01:00
|
|
|
asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
|
|
|
|
: : "r"(va) : "memory");
|
|
|
|
break;
|
|
|
|
default:
|
2012-09-10 04:52:50 +02:00
|
|
|
/* We need 14 to 14 + i bits of va */
|
2013-04-28 11:37:35 +02:00
|
|
|
penc = mmu_psize_defs[psize].penc[apsize];
|
2013-04-28 11:37:39 +02:00
|
|
|
va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
|
2006-06-15 13:15:44 +02:00
|
|
|
va |= penc << 12;
|
2007-10-11 12:37:10 +02:00
|
|
|
va |= ssize << 8;
|
2014-04-21 07:07:36 +02:00
|
|
|
/*
|
|
|
|
* AVAL bits:
|
|
|
|
* We don't need all the bits, but rest of the bits
|
|
|
|
* must be ignored by the processor.
|
|
|
|
* vpn cover upto 65 bits of va. (0...65) and we need
|
|
|
|
* 58..64 bits of va.
|
|
|
|
*/
|
|
|
|
va |= (vpn & 0xfe);
|
2009-04-29 22:58:01 +02:00
|
|
|
va |= 1; /* L */
|
2005-11-07 01:06:55 +01:00
|
|
|
asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
|
|
|
|
: : "r"(va) : "memory");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2013-04-28 11:37:35 +02:00
|
|
|
static inline void tlbie(unsigned long vpn, int psize, int apsize,
|
|
|
|
int ssize, int local)
|
2005-11-07 01:06:55 +01:00
|
|
|
{
|
2014-10-08 10:55:00 +02:00
|
|
|
unsigned int use_local;
|
2011-04-06 21:48:50 +02:00
|
|
|
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2014-10-08 10:55:00 +02:00
|
|
|
use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use();
|
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
if (use_local)
|
|
|
|
use_local = mmu_psize_defs[psize].tlbiel;
|
|
|
|
if (lock_tlbie && !use_local)
|
2010-02-18 03:22:35 +01:00
|
|
|
raw_spin_lock(&native_tlbie_lock);
|
2005-11-07 01:06:55 +01:00
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
if (use_local) {
|
2013-04-28 11:37:35 +02:00
|
|
|
__tlbiel(vpn, psize, apsize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
} else {
|
2013-04-28 11:37:35 +02:00
|
|
|
__tlbie(vpn, psize, apsize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
asm volatile("eieio; tlbsync; ptesync": : :"memory");
|
|
|
|
}
|
|
|
|
if (lock_tlbie && !use_local)
|
2010-02-18 03:22:35 +01:00
|
|
|
raw_spin_unlock(&native_tlbie_lock);
|
2005-11-07 01:06:55 +01:00
|
|
|
}
|
|
|
|
|
2007-06-13 06:52:56 +02:00
|
|
|
static inline void native_lock_hpte(struct hash_pte *hptep)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2013-09-23 04:04:36 +02:00
|
|
|
unsigned long *word = (unsigned long *)&hptep->v;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
while (1) {
|
2010-02-10 02:03:06 +01:00
|
|
|
if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
|
2005-04-17 00:20:36 +02:00
|
|
|
break;
|
|
|
|
while(test_bit(HPTE_LOCK_BIT, word))
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-06-13 06:52:56 +02:00
|
|
|
static inline void native_unlock_hpte(struct hash_pte *hptep)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2013-09-23 04:04:36 +02:00
|
|
|
unsigned long *word = (unsigned long *)&hptep->v;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-02-10 02:03:06 +01:00
|
|
|
clear_bit_unlock(HPTE_LOCK_BIT, word);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned long pa, unsigned long rflags,
|
2013-04-28 11:37:35 +02:00
|
|
|
unsigned long vflags, int psize, int apsize, int ssize)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep = htab_address + hpte_group;
|
2005-07-13 10:11:42 +02:00
|
|
|
unsigned long hpte_v, hpte_r;
|
2005-04-17 00:20:36 +02:00
|
|
|
int i;
|
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
if (!(vflags & HPTE_V_BOLTED)) {
|
2012-09-10 04:52:50 +02:00
|
|
|
DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx,"
|
2005-11-07 01:06:55 +01:00
|
|
|
" rflags=%lx, vflags=%lx, psize=%d)\n",
|
2012-09-10 04:52:50 +02:00
|
|
|
hpte_group, vpn, pa, rflags, vflags, psize);
|
2005-11-07 01:06:55 +01:00
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
for (i = 0; i < HPTES_PER_GROUP; i++) {
|
2013-09-23 04:04:36 +02:00
|
|
|
if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
|
2005-04-17 00:20:36 +02:00
|
|
|
/* retry with lock held */
|
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 04:04:36 +02:00
|
|
|
if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
|
2005-04-17 00:20:36 +02:00
|
|
|
break;
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
}
|
|
|
|
|
|
|
|
hptep++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i == HPTES_PER_GROUP)
|
|
|
|
return -1;
|
|
|
|
|
2013-04-28 11:37:35 +02:00
|
|
|
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
|
2016-04-29 15:25:43 +02:00
|
|
|
hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags;
|
2005-11-07 01:06:55 +01:00
|
|
|
|
|
|
|
if (!(vflags & HPTE_V_BOLTED)) {
|
|
|
|
DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
|
|
|
|
i, hpte_v, hpte_r);
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2013-09-23 04:04:36 +02:00
|
|
|
hptep->r = cpu_to_be64(hpte_r);
|
2005-04-17 00:20:36 +02:00
|
|
|
/* Guarantee the second dword is visible before the valid bit */
|
2007-07-10 06:49:09 +02:00
|
|
|
eieio();
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* Now set the first dword including the valid bit
|
|
|
|
* NOTE: this also unlocks the hpte
|
|
|
|
*/
|
2013-09-23 04:04:36 +02:00
|
|
|
hptep->v = cpu_to_be64(hpte_v);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
__asm__ __volatile__ ("ptesync" : : : "memory");
|
|
|
|
|
2005-07-13 10:11:42 +02:00
|
|
|
return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static long native_hpte_remove(unsigned long hpte_group)
|
|
|
|
{
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep;
|
2005-04-17 00:20:36 +02:00
|
|
|
int i;
|
|
|
|
int slot_offset;
|
2005-07-13 10:11:42 +02:00
|
|
|
unsigned long hpte_v;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
DBG_LOW(" remove(group=%lx)\n", hpte_group);
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/* pick a random entry to start at */
|
|
|
|
slot_offset = mftb() & 0x7;
|
|
|
|
|
|
|
|
for (i = 0; i < HPTES_PER_GROUP; i++) {
|
|
|
|
hptep = htab_address + hpte_group + slot_offset;
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2005-07-13 10:11:42 +02:00
|
|
|
if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
|
2005-04-17 00:20:36 +02:00
|
|
|
/* retry with lock held */
|
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-07-13 10:11:42 +02:00
|
|
|
if ((hpte_v & HPTE_V_VALID)
|
|
|
|
&& !(hpte_v & HPTE_V_BOLTED))
|
2005-04-17 00:20:36 +02:00
|
|
|
break;
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
}
|
|
|
|
|
|
|
|
slot_offset++;
|
|
|
|
slot_offset &= 0x7;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i == HPTES_PER_GROUP)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
/* Invalidate the hpte. NOTE: this also unlocks it */
|
2005-07-13 10:11:42 +02:00
|
|
|
hptep->v = 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
|
2013-06-20 11:00:13 +02:00
|
|
|
unsigned long vpn, int bpsize,
|
2014-12-04 06:30:14 +01:00
|
|
|
int apsize, int ssize, unsigned long flags)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep = htab_address + slot;
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned long hpte_v, want_v;
|
2014-12-04 06:30:14 +01:00
|
|
|
int ret = 0, local = 0;
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2013-06-20 11:00:13 +02:00
|
|
|
want_v = hpte_encode_avpn(vpn, bpsize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
|
|
|
|
vpn, want_v & HPTE_V_AVPN, slot, newpp);
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2013-05-31 03:03:24 +02:00
|
|
|
/*
|
|
|
|
* We need to invalidate the TLB always because hpte_remove doesn't do
|
|
|
|
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
|
|
|
|
* random entry from it. When we do that we don't invalidate the TLB
|
|
|
|
* (hpte_remove) because we assume the old translation is still
|
|
|
|
* technically "valid".
|
|
|
|
*/
|
2013-06-20 11:00:13 +02:00
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
|
2005-11-07 01:06:55 +01:00
|
|
|
DBG_LOW(" -> miss\n");
|
|
|
|
ret = -1;
|
|
|
|
} else {
|
2014-11-03 15:51:34 +01:00
|
|
|
native_lock_hpte(hptep);
|
|
|
|
/* recheck with locks held */
|
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
|
|
|
if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
|
|
|
|
!(hpte_v & HPTE_V_VALID))) {
|
|
|
|
ret = -1;
|
|
|
|
} else {
|
|
|
|
DBG_LOW(" -> hit\n");
|
|
|
|
/* Update the HPTE */
|
|
|
|
hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
|
|
|
|
~(HPTE_R_PP | HPTE_R_N)) |
|
|
|
|
(newpp & (HPTE_R_PP | HPTE_R_N |
|
|
|
|
HPTE_R_C)));
|
|
|
|
}
|
|
|
|
native_unlock_hpte(hptep);
|
2005-11-07 01:06:55 +01:00
|
|
|
}
|
2014-12-04 06:30:14 +01:00
|
|
|
|
|
|
|
if (flags & HPTE_LOCAL_UPDATE)
|
|
|
|
local = 1;
|
|
|
|
/*
|
|
|
|
* Ensure it is out of the tlb too if it is not a nohpte fault
|
|
|
|
*/
|
|
|
|
if (!(flags & HPTE_NOHPTE_UPDATE))
|
|
|
|
tlbie(vpn, bpsize, apsize, ssize, local);
|
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
return ret;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
static long native_hpte_find(unsigned long vpn, int psize, int ssize)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep;
|
2005-04-17 00:20:36 +02:00
|
|
|
unsigned long hash;
|
2007-10-11 12:37:10 +02:00
|
|
|
unsigned long i;
|
2005-04-17 00:20:36 +02:00
|
|
|
long slot;
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned long want_v, hpte_v;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
|
2013-04-28 11:37:34 +02:00
|
|
|
want_v = hpte_encode_avpn(vpn, psize, ssize);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-10-11 12:37:10 +02:00
|
|
|
/* Bolted mappings are only ever in the primary group */
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
for (i = 0; i < HPTES_PER_GROUP; i++) {
|
|
|
|
hptep = htab_address + slot;
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-10-11 12:37:10 +02:00
|
|
|
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
|
|
|
|
/* HPTE matches */
|
|
|
|
return slot;
|
|
|
|
++slot;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the page protection bits. Intended to be used to create
|
|
|
|
* guard pages for kernel data structures on pages which are bolted
|
|
|
|
* in the HPT. Assumes pages being operated on will not be stolen.
|
|
|
|
*
|
|
|
|
* No need to lock here because we should be the only user.
|
|
|
|
*/
|
2005-11-07 01:06:55 +01:00
|
|
|
static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
|
2007-10-11 12:37:10 +02:00
|
|
|
int psize, int ssize)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2012-09-10 04:52:50 +02:00
|
|
|
unsigned long vpn;
|
|
|
|
unsigned long vsid;
|
2005-04-17 00:20:36 +02:00
|
|
|
long slot;
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-10-11 12:37:10 +02:00
|
|
|
vsid = get_kernel_vsid(ea, ssize);
|
2012-09-10 04:52:50 +02:00
|
|
|
vpn = hpt_vpn(ea, vsid, ssize);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
slot = native_hpte_find(vpn, psize, ssize);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (slot == -1)
|
|
|
|
panic("could not find page to bolt\n");
|
|
|
|
hptep = htab_address + slot;
|
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
/* Update the HPTE */
|
2013-09-23 04:04:36 +02:00
|
|
|
hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
|
|
|
|
~(HPTE_R_PP | HPTE_R_N)) |
|
|
|
|
(newpp & (HPTE_R_PP | HPTE_R_N)));
|
2013-06-20 11:00:13 +02:00
|
|
|
/*
|
|
|
|
* Ensure it is out of the tlb too. Bolted entries base and
|
|
|
|
* actual page size will be same.
|
|
|
|
*/
|
|
|
|
tlbie(vpn, psize, psize, ssize, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
|
2013-06-20 11:00:13 +02:00
|
|
|
int bpsize, int apsize, int ssize, int local)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep = htab_address + slot;
|
2005-07-13 10:11:42 +02:00
|
|
|
unsigned long hpte_v;
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned long want_v;
|
2005-04-17 00:20:36 +02:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2013-06-20 11:00:13 +02:00
|
|
|
want_v = hpte_encode_avpn(vpn, bpsize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2013-05-31 03:03:24 +02:00
|
|
|
/*
|
|
|
|
* We need to invalidate the TLB always because hpte_remove doesn't do
|
|
|
|
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
|
|
|
|
* random entry from it. When we do that we don't invalidate the TLB
|
|
|
|
* (hpte_remove) because we assume the old translation is still
|
|
|
|
* technically "valid".
|
|
|
|
*/
|
2013-06-20 11:00:13 +02:00
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
|
2005-04-17 00:20:36 +02:00
|
|
|
native_unlock_hpte(hptep);
|
2005-11-07 01:06:55 +01:00
|
|
|
else
|
2005-04-17 00:20:36 +02:00
|
|
|
/* Invalidate the hpte. NOTE: this also unlocks it */
|
2005-07-13 10:11:42 +02:00
|
|
|
hptep->v = 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
/* Invalidate the TLB */
|
2013-06-20 11:00:13 +02:00
|
|
|
tlbie(vpn, bpsize, apsize, ssize, local);
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2015-12-01 04:36:53 +01:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2014-08-13 09:01:58 +02:00
|
|
|
static void native_hugepage_invalidate(unsigned long vsid,
|
|
|
|
unsigned long addr,
|
2013-06-20 11:00:27 +02:00
|
|
|
unsigned char *hpte_slot_array,
|
2014-11-02 16:45:28 +01:00
|
|
|
int psize, int ssize, int local)
|
2013-06-20 11:00:27 +02:00
|
|
|
{
|
2014-08-13 09:02:01 +02:00
|
|
|
int i;
|
2013-06-20 11:00:27 +02:00
|
|
|
struct hash_pte *hptep;
|
|
|
|
int actual_psize = MMU_PAGE_16M;
|
|
|
|
unsigned int max_hpte_count, valid;
|
|
|
|
unsigned long flags, s_addr = addr;
|
|
|
|
unsigned long hpte_v, want_v, shift;
|
2014-08-13 09:01:58 +02:00
|
|
|
unsigned long hidx, vpn = 0, hash, slot;
|
2013-06-20 11:00:27 +02:00
|
|
|
|
|
|
|
shift = mmu_psize_defs[psize].shift;
|
|
|
|
max_hpte_count = 1U << (PMD_SHIFT - shift);
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
for (i = 0; i < max_hpte_count; i++) {
|
|
|
|
valid = hpte_valid(hpte_slot_array, i);
|
|
|
|
if (!valid)
|
|
|
|
continue;
|
|
|
|
hidx = hpte_hash_index(hpte_slot_array, i);
|
|
|
|
|
|
|
|
/* get the vpn */
|
|
|
|
addr = s_addr + (i * (1ul << shift));
|
|
|
|
vpn = hpt_vpn(addr, vsid, ssize);
|
|
|
|
hash = hpt_hash(vpn, shift, ssize);
|
|
|
|
if (hidx & _PTEIDX_SECONDARY)
|
|
|
|
hash = ~hash;
|
|
|
|
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
slot += hidx & _PTEIDX_GROUP_IX;
|
|
|
|
|
|
|
|
hptep = htab_address + slot;
|
|
|
|
want_v = hpte_encode_avpn(vpn, psize, ssize);
|
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2013-06-20 11:00:27 +02:00
|
|
|
|
|
|
|
/* Even if we miss, we need to invalidate the TLB */
|
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
else
|
|
|
|
/* Invalidate the hpte. NOTE: this also unlocks it */
|
|
|
|
hptep->v = 0;
|
2014-08-13 09:02:01 +02:00
|
|
|
/*
|
|
|
|
* We need to do tlb invalidate for all the address, tlbie
|
|
|
|
* instruction compares entry_VA in tlb with the VA specified
|
|
|
|
* here
|
|
|
|
*/
|
2014-11-02 16:45:28 +01:00
|
|
|
tlbie(vpn, psize, actual_psize, ssize, local);
|
2013-06-20 11:00:27 +02:00
|
|
|
}
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
2015-12-01 04:36:53 +01:00
|
|
|
#else
|
|
|
|
static void native_hugepage_invalidate(unsigned long vsid,
|
|
|
|
unsigned long addr,
|
|
|
|
unsigned char *hpte_slot_array,
|
|
|
|
int psize, int ssize, int local)
|
|
|
|
{
|
|
|
|
WARN(1, "%s called without THP support\n", __func__);
|
|
|
|
}
|
|
|
|
#endif
|
2013-06-20 11:00:27 +02:00
|
|
|
|
2013-06-20 11:00:13 +02:00
|
|
|
static inline int __hpte_actual_psize(unsigned int lp, int psize)
|
|
|
|
{
|
|
|
|
int i, shift;
|
|
|
|
unsigned int mask;
|
|
|
|
|
|
|
|
/* start from 1 ignoring MMU_PAGE_4K */
|
|
|
|
for (i = 1; i < MMU_PAGE_COUNT; i++) {
|
|
|
|
|
|
|
|
/* invalid penc */
|
|
|
|
if (mmu_psize_defs[psize].penc[i] == -1)
|
|
|
|
continue;
|
|
|
|
/*
|
|
|
|
* encoding bits per actual page size
|
|
|
|
* PTE LP actual page size
|
|
|
|
* rrrr rrrz >=8KB
|
|
|
|
* rrrr rrzz >=16KB
|
|
|
|
* rrrr rzzz >=32KB
|
|
|
|
* rrrr zzzz >=64KB
|
|
|
|
* .......
|
|
|
|
*/
|
|
|
|
shift = mmu_psize_defs[i].shift - LP_SHIFT;
|
|
|
|
if (shift > LP_BITS)
|
|
|
|
shift = LP_BITS;
|
|
|
|
mask = (1 << shift) - 1;
|
|
|
|
if ((lp & mask) == mmu_psize_defs[psize].penc[i])
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2007-06-13 06:52:56 +02:00
|
|
|
static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
|
2013-04-28 11:37:35 +02:00
|
|
|
int *psize, int *apsize, int *ssize, unsigned long *vpn)
|
2007-05-02 16:19:11 +02:00
|
|
|
{
|
2012-09-10 04:52:49 +02:00
|
|
|
unsigned long avpn, pteg, vpi;
|
2013-09-23 04:04:36 +02:00
|
|
|
unsigned long hpte_v = be64_to_cpu(hpte->v);
|
|
|
|
unsigned long hpte_r = be64_to_cpu(hpte->r);
|
2012-09-10 04:52:49 +02:00
|
|
|
unsigned long vsid, seg_off;
|
2013-04-28 11:37:36 +02:00
|
|
|
int size, a_size, shift;
|
|
|
|
/* Look at the 8 bit LP value */
|
2013-09-23 04:04:36 +02:00
|
|
|
unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
|
2007-05-02 16:19:11 +02:00
|
|
|
|
2013-04-28 11:37:35 +02:00
|
|
|
if (!(hpte_v & HPTE_V_LARGE)) {
|
|
|
|
size = MMU_PAGE_4K;
|
|
|
|
a_size = MMU_PAGE_4K;
|
|
|
|
} else {
|
2007-05-02 16:19:11 +02:00
|
|
|
for (size = 0; size < MMU_PAGE_COUNT; size++) {
|
2005-11-07 01:06:55 +01:00
|
|
|
|
2007-05-02 16:19:11 +02:00
|
|
|
/* valid entries have a shift value */
|
|
|
|
if (!mmu_psize_defs[size].shift)
|
|
|
|
continue;
|
2013-04-28 11:37:35 +02:00
|
|
|
|
2013-04-28 11:37:36 +02:00
|
|
|
a_size = __hpte_actual_psize(lp, size);
|
|
|
|
if (a_size != -1)
|
|
|
|
break;
|
2007-05-02 16:19:11 +02:00
|
|
|
}
|
|
|
|
}
|
2007-05-10 07:28:44 +02:00
|
|
|
/* This works for all page sizes, and for 256M and 1T segments */
|
2016-05-19 09:54:30 +02:00
|
|
|
if (cpu_has_feature(CPU_FTR_ARCH_300))
|
|
|
|
*ssize = hpte_r >> HPTE_R_3_0_SSIZE_SHIFT;
|
|
|
|
else
|
|
|
|
*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
|
|
|
|
|
2007-05-02 16:19:11 +02:00
|
|
|
shift = mmu_psize_defs[size].shift;
|
|
|
|
|
2012-09-10 04:52:49 +02:00
|
|
|
avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
|
|
|
|
pteg = slot / HPTES_PER_GROUP;
|
|
|
|
if (hpte_v & HPTE_V_SECONDARY)
|
|
|
|
pteg = ~pteg;
|
|
|
|
|
|
|
|
switch (*ssize) {
|
|
|
|
case MMU_SEGSIZE_256M:
|
|
|
|
/* We only have 28 - 23 bits of seg_off in avpn */
|
|
|
|
seg_off = (avpn & 0x1f) << 23;
|
|
|
|
vsid = avpn >> 5;
|
|
|
|
/* We can find more bits from the pteg value */
|
|
|
|
if (shift < 23) {
|
|
|
|
vpi = (vsid ^ pteg) & htab_hash_mask;
|
|
|
|
seg_off |= vpi << shift;
|
|
|
|
}
|
2012-09-10 04:52:50 +02:00
|
|
|
*vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
|
2013-07-03 10:20:03 +02:00
|
|
|
break;
|
2012-09-10 04:52:49 +02:00
|
|
|
case MMU_SEGSIZE_1T:
|
|
|
|
/* We only have 40 - 23 bits of seg_off in avpn */
|
|
|
|
seg_off = (avpn & 0x1ffff) << 23;
|
|
|
|
vsid = avpn >> 17;
|
|
|
|
if (shift < 23) {
|
2007-05-10 07:28:44 +02:00
|
|
|
vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
|
2012-09-10 04:52:49 +02:00
|
|
|
seg_off |= vpi << shift;
|
2007-05-02 16:19:11 +02:00
|
|
|
}
|
2012-09-10 04:52:50 +02:00
|
|
|
*vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
|
2013-07-03 10:20:03 +02:00
|
|
|
break;
|
2012-09-10 04:52:49 +02:00
|
|
|
default:
|
2012-09-10 04:52:50 +02:00
|
|
|
*vpn = size = 0;
|
2005-11-07 01:06:55 +01:00
|
|
|
}
|
2013-04-28 11:37:35 +02:00
|
|
|
*psize = size;
|
|
|
|
*apsize = a_size;
|
2005-11-07 01:06:55 +01:00
|
|
|
}
|
|
|
|
|
2005-06-25 23:58:08 +02:00
|
|
|
/*
|
|
|
|
* clear all mappings on kexec. All cpus are in real mode (or they will
|
|
|
|
* be when they isi), and we are the only one left. We rely on our kernel
|
|
|
|
* mapping being 0xC0's and the hardware ignoring those two real bits.
|
|
|
|
*
|
2015-10-08 02:04:26 +02:00
|
|
|
* This must be called with interrupts disabled.
|
|
|
|
*
|
|
|
|
* Taking the native_tlbie_lock is unsafe here due to the possibility of
|
|
|
|
* lockdep being on. On pre POWER5 hardware, not taking the lock could
|
|
|
|
* cause deadlock. POWER5 and newer not taking the lock is fine. This only
|
|
|
|
* gets called during boot before secondary CPUs have come up and during
|
|
|
|
* crashdump and all bets are off anyway.
|
|
|
|
*
|
2005-06-25 23:58:08 +02:00
|
|
|
* TODO: add batching support when enabled. remember, no dynamic memory here,
|
|
|
|
* athough there is the control page available...
|
|
|
|
*/
|
|
|
|
static void native_hpte_clear(void)
|
|
|
|
{
|
2012-09-10 04:52:50 +02:00
|
|
|
unsigned long vpn = 0;
|
2015-10-08 02:04:26 +02:00
|
|
|
unsigned long slot, slots;
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep = htab_address;
|
2012-09-10 04:52:50 +02:00
|
|
|
unsigned long hpte_v;
|
2005-06-25 23:58:08 +02:00
|
|
|
unsigned long pteg_count;
|
2013-04-28 11:37:35 +02:00
|
|
|
int psize, apsize, ssize;
|
2005-06-25 23:58:08 +02:00
|
|
|
|
|
|
|
pteg_count = htab_hash_mask + 1;
|
|
|
|
|
|
|
|
slots = pteg_count * HPTES_PER_GROUP;
|
|
|
|
|
|
|
|
for (slot = 0; slot < slots; slot++, hptep++) {
|
|
|
|
/*
|
|
|
|
* we could lock the pte here, but we are the only cpu
|
|
|
|
* running, right? and for crash dump, we probably
|
|
|
|
* don't want to wait for a maybe bad cpu.
|
|
|
|
*/
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-06-25 23:58:08 +02:00
|
|
|
|
2006-02-22 17:13:08 +01:00
|
|
|
/*
|
2015-10-08 02:04:26 +02:00
|
|
|
* Call __tlbie() here rather than tlbie() since we can't take the
|
|
|
|
* native_tlbie_lock.
|
2006-02-22 17:13:08 +01:00
|
|
|
*/
|
2005-07-13 10:11:42 +02:00
|
|
|
if (hpte_v & HPTE_V_VALID) {
|
2013-04-28 11:37:35 +02:00
|
|
|
hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
|
2005-07-13 10:11:42 +02:00
|
|
|
hptep->v = 0;
|
2013-04-28 11:37:35 +02:00
|
|
|
__tlbie(vpn, psize, apsize, ssize);
|
2005-06-25 23:58:08 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-22 17:13:08 +01:00
|
|
|
asm volatile("eieio; tlbsync; ptesync":::"memory");
|
2005-06-25 23:58:08 +02:00
|
|
|
}
|
|
|
|
|
2005-11-07 01:06:55 +01:00
|
|
|
/*
|
|
|
|
* Batched hash table flush, we batch the tlbie's to avoid taking/releasing
|
|
|
|
* the lock all the time
|
|
|
|
*/
|
2005-09-20 05:52:50 +02:00
|
|
|
static void native_flush_hash_range(unsigned long number, int local)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2012-09-10 04:52:50 +02:00
|
|
|
unsigned long vpn;
|
|
|
|
unsigned long hash, index, hidx, shift, slot;
|
2007-06-13 06:52:56 +02:00
|
|
|
struct hash_pte *hptep;
|
2005-07-13 10:11:42 +02:00
|
|
|
unsigned long hpte_v;
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned long want_v;
|
|
|
|
unsigned long flags;
|
|
|
|
real_pte_t pte;
|
powerpc: Replace __get_cpu_var uses
This still has not been merged and now powerpc is the only arch that does
not have this change. Sorry about missing linuxppc-dev before.
V2->V2
- Fix up to work against 3.18-rc1
__get_cpu_var() is used for multiple purposes in the kernel source. One of
them is address calculation via the form &__get_cpu_var(x). This calculates
the address for the instance of the percpu variable of the current processor
based on an offset.
Other use cases are for storing and retrieving data from the current
processors percpu area. __get_cpu_var() can be used as an lvalue when
writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
__get_cpu_var() always only does an address determination. However, store
and retrieve operations could use a segment prefix (or global register on
other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a
percpu area and use optimized assembly code to read and write per cpu
variables.
This patch converts __get_cpu_var into either an explicit address
calculation using this_cpu_ptr() or into a use of this_cpu operations that
use the offset. Thereby address calculations are avoided and less registers
are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so
the macro is removed too.
The patch set includes passes over all arches as well. Once these operations
are used throughout then specialized macros can be defined in non -x86
arches as well in order to optimize per cpu access by f.e. using a global
register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);
2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);
3. Retrieve the content of the current processors instance of a per cpu
variable.
DEFINE_PER_CPU(int, y);
int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y);
struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y)
__get_cpu_var(y) = x;
Converts to
__this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y);
__get_cpu_var(y)++
Converts to
__this_cpu_inc(y)
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
CC: Paul Mackerras <paulus@samba.org>
Signed-off-by: Christoph Lameter <cl@linux.com>
[mpe: Fix build errors caused by set/or_softirq_pending(), and rework
assignment in __set_breakpoint() to use memcpy().]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2014-10-21 22:23:25 +02:00
|
|
|
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
|
2005-11-07 01:06:55 +01:00
|
|
|
unsigned long psize = batch->psize;
|
2007-10-11 12:37:10 +02:00
|
|
|
int ssize = batch->ssize;
|
2005-11-07 01:06:55 +01:00
|
|
|
int i;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
for (i = 0; i < number; i++) {
|
2012-09-10 04:52:50 +02:00
|
|
|
vpn = batch->vpn[i];
|
2005-11-07 01:06:55 +01:00
|
|
|
pte = batch->pte[i];
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
|
|
|
|
hash = hpt_hash(vpn, shift, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
hidx = __rpte_to_hidx(pte, index);
|
|
|
|
if (hidx & _PTEIDX_SECONDARY)
|
|
|
|
hash = ~hash;
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
slot += hidx & _PTEIDX_GROUP_IX;
|
|
|
|
hptep = htab_address + slot;
|
2013-04-28 11:37:34 +02:00
|
|
|
want_v = hpte_encode_avpn(vpn, psize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 04:04:36 +02:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-11-07 01:06:55 +01:00
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) ||
|
|
|
|
!(hpte_v & HPTE_V_VALID))
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
else
|
|
|
|
hptep->v = 0;
|
|
|
|
} pte_iterate_hashed_end();
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2011-04-06 21:48:50 +02:00
|
|
|
if (mmu_has_feature(MMU_FTR_TLBIEL) &&
|
2005-11-07 01:06:55 +01:00
|
|
|
mmu_psize_defs[psize].tlbiel && local) {
|
2005-04-17 00:20:36 +02:00
|
|
|
asm volatile("ptesync":::"memory");
|
2005-11-07 01:06:55 +01:00
|
|
|
for (i = 0; i < number; i++) {
|
2012-09-10 04:52:50 +02:00
|
|
|
vpn = batch->vpn[i];
|
2005-11-07 01:06:55 +01:00
|
|
|
pte = batch->pte[i];
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
pte_iterate_hashed_subpages(pte, psize,
|
|
|
|
vpn, index, shift) {
|
2013-04-28 11:37:35 +02:00
|
|
|
__tlbiel(vpn, psize, psize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
} pte_iterate_hashed_end();
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
asm volatile("ptesync":::"memory");
|
|
|
|
} else {
|
2011-04-06 21:48:50 +02:00
|
|
|
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
if (lock_tlbie)
|
2010-02-18 03:22:35 +01:00
|
|
|
raw_spin_lock(&native_tlbie_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
asm volatile("ptesync":::"memory");
|
2005-11-07 01:06:55 +01:00
|
|
|
for (i = 0; i < number; i++) {
|
2012-09-10 04:52:50 +02:00
|
|
|
vpn = batch->vpn[i];
|
2005-11-07 01:06:55 +01:00
|
|
|
pte = batch->pte[i];
|
|
|
|
|
2012-09-10 04:52:50 +02:00
|
|
|
pte_iterate_hashed_subpages(pte, psize,
|
|
|
|
vpn, index, shift) {
|
2013-04-28 11:37:35 +02:00
|
|
|
__tlbie(vpn, psize, psize, ssize);
|
2005-11-07 01:06:55 +01:00
|
|
|
} pte_iterate_hashed_end();
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
asm volatile("eieio; tlbsync; ptesync":::"memory");
|
|
|
|
|
|
|
|
if (lock_tlbie)
|
2010-02-18 03:22:35 +01:00
|
|
|
raw_spin_unlock(&native_tlbie_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2016-04-29 15:25:43 +02:00
|
|
|
static int native_update_partition_table(u64 patb1)
|
|
|
|
{
|
|
|
|
partition_tb->patb1 = cpu_to_be64(patb1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-06-23 10:16:38 +02:00
|
|
|
void __init hpte_init_native(void)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
ppc_md.hpte_invalidate = native_hpte_invalidate;
|
|
|
|
ppc_md.hpte_updatepp = native_hpte_updatepp;
|
|
|
|
ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
|
|
|
|
ppc_md.hpte_insert = native_hpte_insert;
|
2005-06-25 23:58:08 +02:00
|
|
|
ppc_md.hpte_remove = native_hpte_remove;
|
|
|
|
ppc_md.hpte_clear_all = native_hpte_clear;
|
2012-09-21 00:08:28 +02:00
|
|
|
ppc_md.flush_hash_range = native_flush_hash_range;
|
2013-06-20 11:00:27 +02:00
|
|
|
ppc_md.hugepage_invalidate = native_hugepage_invalidate;
|
2016-04-29 15:25:43 +02:00
|
|
|
|
|
|
|
if (cpu_has_feature(CPU_FTR_ARCH_300))
|
|
|
|
ppc_md.update_partition_table = native_update_partition_table;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|