powerpc/8xx: Make pinning of ITLBs optional

As stated in a comment in head_8xx.S, today we "Always pin the first
8 MB ITLB to prevent ITLB misses while mucking around with SRR0/SRR1
in asm".

This issue has just been cleared by the preceding patch, therefore
we can make this pinning optional (on by default) and independent
of DATA pinning.

This patch also makes pinning of IMMR independent of pinning of DATA.

Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
This commit is contained in:
Christophe Leroy 2017-07-12 12:08:51 +02:00 committed by Michael Ellerman
parent 0eb0d2e77d
commit a3059b0ca0
3 changed files with 62 additions and 13 deletions

View File

@ -1178,10 +1178,20 @@ config PIN_TLB
bool "Pinned Kernel TLBs (860 ONLY)"
depends on ADVANCED_OPTIONS && PPC_8xx
config PIN_TLB_DATA
bool "Pinned TLB for DATA"
depends on PIN_TLB
default y
config PIN_TLB_IMMR
bool "Pinned TLB for IMMR"
depends on PIN_TLB
default y
config PIN_TLB_TEXT
bool "Pinned TLB for TEXT"
depends on PIN_TLB
default y
endmenu
if PPC64

View File

@ -55,6 +55,15 @@
#define SIMPLE_KERNEL_ADDRESS 1
#endif
/*
* We need an ITLB miss handler for kernel addresses if:
* - Either we have modules
* - Or we have not pinned the first 8M
*/
#if defined(CONFIG_MODULES) || !defined(CONFIG_PIN_TLB_TEXT) || \
defined(CONFIG_DEBUG_PAGEALLOC)
#define ITLB_MISS_KERNEL 1
#endif
/*
* Value for the bits that have fixed value in RPN entries.
@ -317,7 +326,7 @@ SystemCall:
#endif
InstructionTLBMiss:
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mtspr SPRN_SPRG_SCRATCH2, r3
#endif
EXCEPTION_PROLOG_0
@ -335,23 +344,31 @@ InstructionTLBMiss:
INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
/* Only modules will cause ITLB Misses as we always
* pin the first 8MB of kernel memory */
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mfcr r3
#endif
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
#ifdef SIMPLE_KERNEL_ADDRESS
#ifdef ITLB_MISS_KERNEL
#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
#else
rlwinm r11, r10, 16, 0xfff8
cmpli cr0, r11, PAGE_OFFSET@h
#ifndef CONFIG_PIN_TLB_TEXT
/* It is assumed that kernel code fits into the first 8M page */
_ENTRY(ITLBMiss_cmp)
cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h
#endif
#endif
#endif
mfspr r11, SPRN_M_TW /* Get level 1 table */
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
#ifdef SIMPLE_KERNEL_ADDRESS
#ifdef ITLB_MISS_KERNEL
#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
beq+ 3f
#else
blt+ 3f
#endif
#ifndef CONFIG_PIN_TLB_TEXT
blt cr7, ITLBMissLinear
#endif
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
3:
@ -370,7 +387,7 @@ InstructionTLBMiss:
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
lwz r10, 0(r10) /* Get the pte */
4:
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mtcr r3
#endif
/* Insert the APG into the TWC from the Linux PTE. */
@ -401,7 +418,7 @@ InstructionTLBMiss:
MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */
/* Restore registers */
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mfspr r3, SPRN_SPRG_SCRATCH2
#endif
EXCEPTION_EPILOG_0
@ -696,6 +713,22 @@ DTLBMissLinear:
EXCEPTION_EPILOG_0
rfi
#ifndef CONFIG_PIN_TLB_TEXT
ITLBMissLinear:
mtcr r3
/* Set 8M byte page and mark it valid */
li r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC
MTSPR_CPU6(SPRN_MI_TWC, r11, r3)
rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \
_PAGE_PRESENT
MTSPR_CPU6(SPRN_MI_RPN, r10, r11) /* Update TLB entry */
mfspr r3, SPRN_SPRG_SCRATCH2
EXCEPTION_EPILOG_0
rfi
#endif
/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
* by decoding the registers used by the dcbx instruction and adding them.
* DAR is set to the calculated address.
@ -955,15 +988,14 @@ initial_mmu:
mtspr SPRN_MD_CTR, r10 /* remove PINNED DTLB entries */
tlbia /* Invalidate all TLB entries */
/* Always pin the first 8 MB ITLB to prevent ITLB
misses while mucking around with SRR0/SRR1 in asm
*/
#ifdef CONFIG_PIN_TLB_TEXT
lis r8, MI_RSV4I@h
ori r8, r8, 0x1c00
mtspr SPRN_MI_CTR, r8 /* Set instruction MMU control */
#endif
#ifdef CONFIG_PIN_TLB
#ifdef CONFIG_PIN_TLB_DATA
oris r10, r10, MD_RSV4I@h
mtspr SPRN_MD_CTR, r10 /* Set data TLB control */
#endif
@ -989,6 +1021,7 @@ initial_mmu:
* internal registers (among other things).
*/
#ifdef CONFIG_PIN_TLB_IMMR
oris r10, r10, MD_RSV4I@h
ori r10, r10, 0x1c00
mtspr SPRN_MD_CTR, r10

View File

@ -65,7 +65,7 @@ unsigned long p_block_mapped(phys_addr_t pa)
void __init MMU_init_hw(void)
{
/* PIN up to the 3 first 8Mb after IMMR in DTLB table */
#ifdef CONFIG_PIN_TLB
#ifdef CONFIG_PIN_TLB_DATA
unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
#ifdef CONFIG_PIN_TLB_IMMR
@ -103,6 +103,9 @@ static void mmu_mapin_immr(void)
extern unsigned int DTLBMiss_jmp;
#endif
extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;
#ifndef CONFIG_PIN_TLB_TEXT
extern unsigned int ITLBMiss_cmp;
#endif
void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)
{
@ -122,6 +125,9 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
mmu_mapin_immr();
#ifndef CONFIG_PIN_TLB_IMMR
patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
#endif
#ifndef CONFIG_PIN_TLB_TEXT
mmu_patch_cmp_limit(&ITLBMiss_cmp, 0);
#endif
} else {
mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);