2006-02-01 03:29:18 +01:00
|
|
|
/* arch/sparc64/mm/tsb.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <asm/system.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/tlb.h>
|
2006-02-01 03:31:06 +01:00
|
|
|
#include <asm/mmu_context.h>
|
2006-02-01 03:31:20 +01:00
|
|
|
#include <asm/pgtable.h>
|
2006-02-01 03:31:38 +01:00
|
|
|
#include <asm/tsb.h>
|
2006-03-19 03:12:42 +01:00
|
|
|
#include <asm/oplib.h>
|
2006-02-01 03:29:18 +01:00
|
|
|
|
|
|
|
extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries)
|
2006-02-01 03:29:18 +01:00
|
|
|
{
|
2006-03-22 09:49:59 +01:00
|
|
|
vaddr >>= hash_shift;
|
2006-02-01 03:31:20 +01:00
|
|
|
return vaddr & (nentries - 1);
|
2006-02-01 03:29:18 +01:00
|
|
|
}
|
|
|
|
|
2006-02-18 03:01:02 +01:00
|
|
|
static inline int tag_compare(unsigned long tag, unsigned long vaddr)
|
2006-02-01 03:29:18 +01:00
|
|
|
{
|
2006-02-18 03:01:02 +01:00
|
|
|
return (tag == (vaddr >> 22));
|
2006-02-01 03:29:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* TSB flushes need only occur on the processor initiating the address
|
|
|
|
* space modification, not on each cpu the address space has run on.
|
|
|
|
* Only the TLB flush needs that treatment.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void flush_tsb_kernel_range(unsigned long start, unsigned long end)
|
|
|
|
{
|
|
|
|
unsigned long v;
|
|
|
|
|
|
|
|
for (v = start; v < end; v += PAGE_SIZE) {
|
2006-03-22 09:49:59 +01:00
|
|
|
unsigned long hash = tsb_hash(v, PAGE_SHIFT,
|
|
|
|
KERNEL_TSB_NENTRIES);
|
2006-02-01 03:31:20 +01:00
|
|
|
struct tsb *ent = &swapper_tsb[hash];
|
2006-02-01 03:29:18 +01:00
|
|
|
|
2006-02-18 03:01:02 +01:00
|
|
|
if (tag_compare(ent->tag, v)) {
|
|
|
|
ent->tag = (1UL << TSB_TAG_INVALID_BIT);
|
2006-02-01 03:29:18 +01:00
|
|
|
membar_storeload_storestore();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
static void __flush_tsb_one(struct mmu_gather *mp, unsigned long hash_shift, unsigned long tsb, unsigned long nentries)
|
2006-02-01 03:29:18 +01:00
|
|
|
{
|
2006-03-22 09:49:59 +01:00
|
|
|
unsigned long i;
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
|
2006-02-01 03:29:18 +01:00
|
|
|
for (i = 0; i < mp->tlb_nr; i++) {
|
|
|
|
unsigned long v = mp->vaddrs[i];
|
2006-02-02 00:55:21 +01:00
|
|
|
unsigned long tag, ent, hash;
|
2006-02-01 03:29:18 +01:00
|
|
|
|
|
|
|
v &= ~0x1UL;
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
hash = tsb_hash(v, hash_shift, nentries);
|
|
|
|
ent = tsb + (hash * sizeof(struct tsb));
|
2006-02-18 03:01:02 +01:00
|
|
|
tag = (v >> 22UL);
|
2006-02-02 00:55:21 +01:00
|
|
|
|
|
|
|
tsb_flush(ent, tag);
|
2006-02-01 03:29:18 +01:00
|
|
|
}
|
2006-03-22 09:49:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void flush_tsb_user(struct mmu_gather *mp)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = mp->mm;
|
|
|
|
unsigned long nentries, base, flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&mm->context.lock, flags);
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
|
|
|
|
nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
|
|
|
|
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
|
|
|
|
base = __pa(base);
|
|
|
|
__flush_tsb_one(mp, PAGE_SHIFT, base, nentries);
|
|
|
|
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
|
|
|
|
base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
|
|
|
|
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
|
|
|
|
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
|
|
|
|
base = __pa(base);
|
|
|
|
__flush_tsb_one(mp, HPAGE_SHIFT, base, nentries);
|
|
|
|
}
|
|
|
|
#endif
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
spin_unlock_irqrestore(&mm->context.lock, flags);
|
2006-02-01 03:29:18 +01:00
|
|
|
}
|
2006-02-01 03:31:06 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB)
|
|
|
|
#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K
|
|
|
|
#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K
|
|
|
|
#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB)
|
|
|
|
#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K
|
|
|
|
#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K
|
|
|
|
#elif defined(CONFIG_SPARC64_PAGE_SIZE_512KB)
|
|
|
|
#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_512K
|
|
|
|
#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_512K
|
|
|
|
#elif defined(CONFIG_SPARC64_PAGE_SIZE_4MB)
|
|
|
|
#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_4MB
|
|
|
|
#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_4MB
|
|
|
|
#else
|
|
|
|
#error Broken base page size setting...
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
#if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
|
|
|
|
#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K
|
|
|
|
#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K
|
|
|
|
#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
|
|
|
|
#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K
|
|
|
|
#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K
|
|
|
|
#elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
|
|
|
|
#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB
|
|
|
|
#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB
|
|
|
|
#else
|
|
|
|
#error Broken huge page size setting...
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes)
|
2006-02-01 03:31:20 +01:00
|
|
|
{
|
|
|
|
unsigned long tsb_reg, base, tsb_paddr;
|
|
|
|
unsigned long page_sz, tte;
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
mm->context.tsb_block[tsb_idx].tsb_nentries =
|
|
|
|
tsb_bytes / sizeof(struct tsb);
|
2006-02-01 03:31:20 +01:00
|
|
|
|
|
|
|
base = TSBMAP_BASE;
|
2006-02-12 06:57:54 +01:00
|
|
|
tte = pgprot_val(PAGE_KERNEL_LOCKED);
|
2006-03-22 09:49:59 +01:00
|
|
|
tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb);
|
2006-02-02 00:55:21 +01:00
|
|
|
BUG_ON(tsb_paddr & (tsb_bytes - 1UL));
|
2006-02-01 03:31:20 +01:00
|
|
|
|
|
|
|
/* Use the smallest page size that can map the whole TSB
|
|
|
|
* in one TLB entry.
|
|
|
|
*/
|
|
|
|
switch (tsb_bytes) {
|
|
|
|
case 8192 << 0:
|
|
|
|
tsb_reg = 0x0UL;
|
|
|
|
#ifdef DCACHE_ALIASING_POSSIBLE
|
|
|
|
base += (tsb_paddr & 8192);
|
|
|
|
#endif
|
|
|
|
page_sz = 8192;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 1:
|
|
|
|
tsb_reg = 0x1UL;
|
|
|
|
page_sz = 64 * 1024;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 2:
|
|
|
|
tsb_reg = 0x2UL;
|
|
|
|
page_sz = 64 * 1024;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 3:
|
|
|
|
tsb_reg = 0x3UL;
|
|
|
|
page_sz = 64 * 1024;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 4:
|
|
|
|
tsb_reg = 0x4UL;
|
|
|
|
page_sz = 512 * 1024;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 5:
|
|
|
|
tsb_reg = 0x5UL;
|
|
|
|
page_sz = 512 * 1024;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 6:
|
|
|
|
tsb_reg = 0x6UL;
|
|
|
|
page_sz = 512 * 1024;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case 8192 << 7:
|
|
|
|
tsb_reg = 0x7UL;
|
|
|
|
page_sz = 4 * 1024 * 1024;
|
|
|
|
break;
|
2006-02-01 03:31:38 +01:00
|
|
|
|
|
|
|
default:
|
|
|
|
BUG();
|
2006-02-01 03:31:20 +01:00
|
|
|
};
|
2006-02-12 06:57:54 +01:00
|
|
|
tte |= pte_sz_bits(page_sz);
|
2006-02-01 03:31:20 +01:00
|
|
|
|
2006-02-10 02:21:53 +01:00
|
|
|
if (tlb_type == cheetah_plus || tlb_type == hypervisor) {
|
2006-02-02 00:55:21 +01:00
|
|
|
/* Physical mapping, no locked TLB entry for TSB. */
|
|
|
|
tsb_reg |= tsb_paddr;
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
|
|
|
|
mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0;
|
|
|
|
mm->context.tsb_block[tsb_idx].tsb_map_pte = 0;
|
2006-02-02 00:55:21 +01:00
|
|
|
} else {
|
|
|
|
tsb_reg |= base;
|
|
|
|
tsb_reg |= (tsb_paddr & (page_sz - 1UL));
|
|
|
|
tte |= (tsb_paddr & ~(page_sz - 1UL));
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
|
|
|
|
mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base;
|
|
|
|
mm->context.tsb_block[tsb_idx].tsb_map_pte = tte;
|
2006-02-02 00:55:21 +01:00
|
|
|
}
|
2006-02-01 03:31:20 +01:00
|
|
|
|
2006-02-10 02:21:53 +01:00
|
|
|
/* Setup the Hypervisor TSB descriptor. */
|
|
|
|
if (tlb_type == hypervisor) {
|
2006-03-22 09:49:59 +01:00
|
|
|
struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx];
|
2006-02-10 02:21:53 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
switch (tsb_idx) {
|
|
|
|
case MM_TSB_BASE:
|
|
|
|
hp->pgsz_idx = HV_PGSZ_IDX_BASE;
|
2006-02-10 02:21:53 +01:00
|
|
|
break;
|
2006-03-22 09:49:59 +01:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
case MM_TSB_HUGE:
|
|
|
|
hp->pgsz_idx = HV_PGSZ_IDX_HUGE;
|
2006-02-10 02:21:53 +01:00
|
|
|
break;
|
2006-03-22 09:49:59 +01:00
|
|
|
#endif
|
|
|
|
default:
|
|
|
|
BUG();
|
2006-02-10 02:21:53 +01:00
|
|
|
};
|
|
|
|
hp->assoc = 1;
|
|
|
|
hp->num_ttes = tsb_bytes / 16;
|
|
|
|
hp->ctx_idx = 0;
|
2006-03-22 09:49:59 +01:00
|
|
|
switch (tsb_idx) {
|
|
|
|
case MM_TSB_BASE:
|
|
|
|
hp->pgsz_mask = HV_PGSZ_MASK_BASE;
|
2006-02-10 02:21:53 +01:00
|
|
|
break;
|
2006-03-22 09:49:59 +01:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
case MM_TSB_HUGE:
|
|
|
|
hp->pgsz_mask = HV_PGSZ_MASK_HUGE;
|
2006-02-10 02:21:53 +01:00
|
|
|
break;
|
2006-03-22 09:49:59 +01:00
|
|
|
#endif
|
|
|
|
default:
|
|
|
|
BUG();
|
2006-02-10 02:21:53 +01:00
|
|
|
};
|
|
|
|
hp->tsb_base = tsb_paddr;
|
|
|
|
hp->resv = 0;
|
|
|
|
}
|
2006-02-01 03:31:20 +01:00
|
|
|
}
|
|
|
|
|
2006-12-07 05:33:20 +01:00
|
|
|
static struct kmem_cache *tsb_caches[8] __read_mostly;
|
2006-03-19 03:12:42 +01:00
|
|
|
|
|
|
|
static const char *tsb_cache_names[8] = {
|
|
|
|
"tsb_8KB",
|
|
|
|
"tsb_16KB",
|
|
|
|
"tsb_32KB",
|
|
|
|
"tsb_64KB",
|
|
|
|
"tsb_128KB",
|
|
|
|
"tsb_256KB",
|
|
|
|
"tsb_512KB",
|
|
|
|
"tsb_1MB",
|
|
|
|
};
|
|
|
|
|
2007-05-06 23:49:51 +02:00
|
|
|
void __init pgtable_cache_init(void)
|
2006-03-19 03:12:42 +01:00
|
|
|
{
|
|
|
|
unsigned long i;
|
|
|
|
|
|
|
|
for (i = 0; i < 8; i++) {
|
|
|
|
unsigned long size = 8192 << i;
|
|
|
|
const char *name = tsb_cache_names[i];
|
|
|
|
|
|
|
|
tsb_caches[i] = kmem_cache_create(name,
|
|
|
|
size, size,
|
2007-05-06 23:49:58 +02:00
|
|
|
0,
|
2006-03-19 03:12:42 +01:00
|
|
|
NULL, NULL);
|
|
|
|
if (!tsb_caches[i]) {
|
|
|
|
prom_printf("Could not create %s cache\n", name);
|
|
|
|
prom_halt();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
/* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
|
|
|
|
* do_sparc64_fault() invokes this routine to try and grow it.
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
*
|
2006-02-01 03:31:38 +01:00
|
|
|
* When we reach the maximum TSB size supported, we stick ~0UL into
|
2006-03-22 09:49:59 +01:00
|
|
|
* tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
|
2006-02-01 03:31:38 +01:00
|
|
|
* will not trigger any longer.
|
|
|
|
*
|
|
|
|
* The TSB can be anywhere from 8K to 1MB in size, in increasing powers
|
|
|
|
* of two. The TSB must be aligned to it's size, so f.e. a 512K TSB
|
2006-03-18 08:40:47 +01:00
|
|
|
* must be 512K aligned. It also must be physically contiguous, so we
|
|
|
|
* cannot use vmalloc().
|
2006-02-01 03:31:38 +01:00
|
|
|
*
|
|
|
|
* The idea here is to grow the TSB when the RSS of the process approaches
|
|
|
|
* the number of entries that the current TSB can hold at once. Currently,
|
|
|
|
* we trigger when the RSS hits 3/4 of the TSB capacity.
|
|
|
|
*/
|
2006-03-22 09:49:59 +01:00
|
|
|
void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss)
|
2006-02-01 03:31:38 +01:00
|
|
|
{
|
|
|
|
unsigned long max_tsb_size = 1 * 1024 * 1024;
|
2006-03-19 03:12:42 +01:00
|
|
|
unsigned long new_size, old_size, flags;
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
struct tsb *old_tsb, *new_tsb;
|
2006-03-19 03:12:42 +01:00
|
|
|
unsigned long new_cache_index, old_cache_index;
|
|
|
|
unsigned long new_rss_limit;
|
2006-03-18 08:40:47 +01:00
|
|
|
gfp_t gfp_flags;
|
2006-02-01 03:31:38 +01:00
|
|
|
|
|
|
|
if (max_tsb_size > (PAGE_SIZE << MAX_ORDER))
|
|
|
|
max_tsb_size = (PAGE_SIZE << MAX_ORDER);
|
|
|
|
|
2006-03-19 03:12:42 +01:00
|
|
|
new_cache_index = 0;
|
|
|
|
for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) {
|
|
|
|
unsigned long n_entries = new_size / sizeof(struct tsb);
|
2006-02-01 03:31:38 +01:00
|
|
|
|
|
|
|
n_entries = (n_entries * 3) / 4;
|
|
|
|
if (n_entries > rss)
|
|
|
|
break;
|
2006-03-19 03:12:42 +01:00
|
|
|
|
|
|
|
new_cache_index++;
|
2006-02-01 03:31:38 +01:00
|
|
|
}
|
|
|
|
|
2006-03-19 03:12:42 +01:00
|
|
|
if (new_size == max_tsb_size)
|
2006-03-18 08:40:47 +01:00
|
|
|
new_rss_limit = ~0UL;
|
|
|
|
else
|
2006-03-19 03:12:42 +01:00
|
|
|
new_rss_limit = ((new_size / sizeof(struct tsb)) * 3) / 4;
|
2006-03-18 08:40:47 +01:00
|
|
|
|
2006-03-19 03:12:42 +01:00
|
|
|
retry_tsb_alloc:
|
2006-03-18 08:40:47 +01:00
|
|
|
gfp_flags = GFP_KERNEL;
|
2006-03-19 03:12:42 +01:00
|
|
|
if (new_size > (PAGE_SIZE * 2))
|
2006-03-18 08:40:47 +01:00
|
|
|
gfp_flags = __GFP_NOWARN | __GFP_NORETRY;
|
|
|
|
|
2006-03-19 03:12:42 +01:00
|
|
|
new_tsb = kmem_cache_alloc(tsb_caches[new_cache_index], gfp_flags);
|
|
|
|
if (unlikely(!new_tsb)) {
|
2006-03-18 08:40:47 +01:00
|
|
|
/* Not being able to fork due to a high-order TSB
|
|
|
|
* allocation failure is very bad behavior. Just back
|
|
|
|
* down to a 0-order allocation and force no TSB
|
|
|
|
* growing for this address space.
|
|
|
|
*/
|
2006-03-22 09:49:59 +01:00
|
|
|
if (mm->context.tsb_block[tsb_index].tsb == NULL &&
|
|
|
|
new_cache_index > 0) {
|
2006-03-19 03:12:42 +01:00
|
|
|
new_cache_index = 0;
|
|
|
|
new_size = 8192;
|
2006-03-18 08:40:47 +01:00
|
|
|
new_rss_limit = ~0UL;
|
2006-03-19 03:12:42 +01:00
|
|
|
goto retry_tsb_alloc;
|
2006-03-18 08:40:47 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* If we failed on a TSB grow, we are under serious
|
|
|
|
* memory pressure so don't try to grow any more.
|
|
|
|
*/
|
2006-03-22 09:49:59 +01:00
|
|
|
if (mm->context.tsb_block[tsb_index].tsb != NULL)
|
|
|
|
mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL;
|
2006-02-01 03:31:38 +01:00
|
|
|
return;
|
2006-03-18 08:40:47 +01:00
|
|
|
}
|
2006-02-01 03:31:38 +01:00
|
|
|
|
2006-02-18 03:01:02 +01:00
|
|
|
/* Mark all tags as invalid. */
|
2006-03-19 08:55:11 +01:00
|
|
|
tsb_init(new_tsb, new_size);
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
|
|
|
|
/* Ok, we are about to commit the changes. If we are
|
|
|
|
* growing an existing TSB the locking is very tricky,
|
|
|
|
* so WATCH OUT!
|
|
|
|
*
|
|
|
|
* We have to hold mm->context.lock while committing to the
|
|
|
|
* new TSB, this synchronizes us with processors in
|
|
|
|
* flush_tsb_user() and switch_mm() for this address space.
|
|
|
|
*
|
|
|
|
* But even with that lock held, processors run asynchronously
|
|
|
|
* accessing the old TSB via TLB miss handling. This is OK
|
|
|
|
* because those actions are just propagating state from the
|
|
|
|
* Linux page tables into the TSB, page table mappings are not
|
|
|
|
* being changed. If a real fault occurs, the processor will
|
|
|
|
* synchronize with us when it hits flush_tsb_user(), this is
|
|
|
|
* also true for the case where vmscan is modifying the page
|
|
|
|
* tables. The only thing we need to be careful with is to
|
|
|
|
* skip any locked TSB entries during copy_tsb().
|
|
|
|
*
|
|
|
|
* When we finish committing to the new TSB, we have to drop
|
|
|
|
* the lock and ask all other cpus running this address space
|
|
|
|
* to run tsb_context_switch() to see the new TSB table.
|
|
|
|
*/
|
|
|
|
spin_lock_irqsave(&mm->context.lock, flags);
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
old_tsb = mm->context.tsb_block[tsb_index].tsb;
|
|
|
|
old_cache_index =
|
|
|
|
(mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL);
|
|
|
|
old_size = (mm->context.tsb_block[tsb_index].tsb_nentries *
|
|
|
|
sizeof(struct tsb));
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
|
2006-03-19 03:12:42 +01:00
|
|
|
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
/* Handle multiple threads trying to grow the TSB at the same time.
|
|
|
|
* One will get in here first, and bump the size and the RSS limit.
|
|
|
|
* The others will get in here next and hit this check.
|
|
|
|
*/
|
2006-03-22 09:49:59 +01:00
|
|
|
if (unlikely(old_tsb &&
|
|
|
|
(rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) {
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
spin_unlock_irqrestore(&mm->context.lock, flags);
|
|
|
|
|
2006-03-19 03:12:42 +01:00
|
|
|
kmem_cache_free(tsb_caches[new_cache_index], new_tsb);
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
return;
|
|
|
|
}
|
2006-02-18 03:01:02 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit;
|
2006-02-01 03:31:38 +01:00
|
|
|
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
if (old_tsb) {
|
|
|
|
extern void copy_tsb(unsigned long old_tsb_base,
|
|
|
|
unsigned long old_tsb_size,
|
|
|
|
unsigned long new_tsb_base,
|
|
|
|
unsigned long new_tsb_size);
|
|
|
|
unsigned long old_tsb_base = (unsigned long) old_tsb;
|
|
|
|
unsigned long new_tsb_base = (unsigned long) new_tsb;
|
|
|
|
|
|
|
|
if (tlb_type == cheetah_plus || tlb_type == hypervisor) {
|
|
|
|
old_tsb_base = __pa(old_tsb_base);
|
|
|
|
new_tsb_base = __pa(new_tsb_base);
|
|
|
|
}
|
2006-03-19 03:12:42 +01:00
|
|
|
copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size);
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
}
|
2006-02-01 03:31:38 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
mm->context.tsb_block[tsb_index].tsb = new_tsb;
|
|
|
|
setup_tsb_params(mm, tsb_index, new_size);
|
2006-02-01 03:31:38 +01:00
|
|
|
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
spin_unlock_irqrestore(&mm->context.lock, flags);
|
|
|
|
|
2006-02-01 03:31:38 +01:00
|
|
|
/* If old_tsb is NULL, we're being invoked for the first time
|
|
|
|
* from init_new_context().
|
|
|
|
*/
|
|
|
|
if (old_tsb) {
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
/* Reload it on the local cpu. */
|
2006-02-01 03:31:38 +01:00
|
|
|
tsb_context_switch(mm);
|
|
|
|
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
/* Now force other processors to do the same. */
|
|
|
|
smp_tsb_sync(mm);
|
|
|
|
|
|
|
|
/* Now it is safe to free the old tsb. */
|
2006-03-19 03:12:42 +01:00
|
|
|
kmem_cache_free(tsb_caches[old_cache_index], old_tsb);
|
2006-02-01 03:31:38 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-02-01 03:31:06 +01:00
|
|
|
int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
|
|
|
|
{
|
2006-03-22 09:49:59 +01:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
unsigned long huge_pte_count;
|
|
|
|
#endif
|
|
|
|
unsigned int i;
|
|
|
|
|
2006-03-07 04:59:50 +01:00
|
|
|
spin_lock_init(&mm->context.lock);
|
2006-02-01 03:31:06 +01:00
|
|
|
|
|
|
|
mm->context.sparc64_ctx_val = 0UL;
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
/* We reset it to zero because the fork() page copying
|
|
|
|
* will re-increment the counters as the parent PTEs are
|
|
|
|
* copied into the child address space.
|
|
|
|
*/
|
|
|
|
huge_pte_count = mm->context.huge_pte_count;
|
|
|
|
mm->context.huge_pte_count = 0;
|
|
|
|
#endif
|
|
|
|
|
2006-02-01 03:31:38 +01:00
|
|
|
/* copy_mm() copies over the parent's mm_struct before calling
|
|
|
|
* us, so we need to zero out the TSB pointer or else tsb_grow()
|
|
|
|
* will be confused and think there is an older TSB to free up.
|
|
|
|
*/
|
2006-03-22 09:49:59 +01:00
|
|
|
for (i = 0; i < MM_NUM_TSBS; i++)
|
|
|
|
mm->context.tsb_block[i].tsb = NULL;
|
[SPARC64]: Fix and re-enable dynamic TSB sizing.
This is good for up to %50 performance improvement of some test cases.
The problem has been the race conditions, and hopefully I've plugged
them all up here.
1) There was a serious race in switch_mm() wrt. lazy TLB
switching to and from kernel threads.
We could erroneously skip a tsb_context_switch() and thus
use a stale TSB across a TSB grow event.
There is a big comment now in that function describing
exactly how it can happen.
2) All code paths that do something with the TSB need to be
guarded with the mm->context.lock spinlock. This makes
page table flushing paths properly synchronize with both
TSB growing and TLB context changes.
3) TSB growing events are moved to the end of successful fault
processing. Previously it was in update_mmu_cache() but
that is deadlock prone. At the end of do_sparc64_fault()
we hold no spinlocks that could deadlock the TSB grow
sequence. We also have dropped the address space semaphore.
While we're here, add prefetching to the copy_tsb() routine
and put it in assembler into the tsb.S file. This piece of
code is quite time critical.
There are some small negative side effects to this code which
can be improved upon. In particular we grab the mm->context.lock
even for the tsb insert done by update_mmu_cache() now and that's
a bit excessive. We can get rid of that locking, and the same
lock taking in flush_tsb_user(), by disabling PSTATE_IE around
the whole operation including the capturing of the tsb pointer
and tsb_nentries value. That would work because anyone growing
the TSB won't free up the old TSB until all cpus respond to the
TSB change cross call.
I'm not quite so confident in that optimization to put it in
right now, but eventually we might be able to and the description
is here for reference.
This code seems very solid now. It passes several parallel GCC
bootstrap builds, and our favorite "nut cruncher" stress test which is
a full "make -j8192" build of a "make allmodconfig" kernel. That puts
about 256 processes on each cpu's run queue, makes lots of process cpu
migrations occur, causes lots of page table and TLB flushing activity,
incurs many context version number changes, and it swaps the machine
real far out to disk even though there is 16GB of ram on this test
system. :-)
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-03-16 11:02:32 +01:00
|
|
|
|
|
|
|
/* If this is fork, inherit the parent's TSB size. We would
|
|
|
|
* grow it to that size on the first page fault anyways.
|
|
|
|
*/
|
2006-03-22 09:49:59 +01:00
|
|
|
tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
|
2006-02-01 03:31:38 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
if (unlikely(huge_pte_count))
|
|
|
|
tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))
|
2006-02-01 03:31:38 +01:00
|
|
|
return -ENOMEM;
|
2006-02-01 03:31:06 +01:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
static void tsb_destroy_one(struct tsb_config *tp)
|
2006-02-01 03:31:06 +01:00
|
|
|
{
|
2006-03-22 09:49:59 +01:00
|
|
|
unsigned long cache_index;
|
2006-02-01 03:31:38 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
if (!tp->tsb)
|
|
|
|
return;
|
|
|
|
cache_index = tp->tsb_reg_val & 0x7UL;
|
|
|
|
kmem_cache_free(tsb_caches[cache_index], tp->tsb);
|
|
|
|
tp->tsb = NULL;
|
|
|
|
tp->tsb_reg_val = 0UL;
|
|
|
|
}
|
2006-02-01 03:31:20 +01:00
|
|
|
|
2006-03-22 09:49:59 +01:00
|
|
|
void destroy_context(struct mm_struct *mm)
|
|
|
|
{
|
|
|
|
unsigned long flags, i;
|
|
|
|
|
|
|
|
for (i = 0; i < MM_NUM_TSBS; i++)
|
|
|
|
tsb_destroy_one(&mm->context.tsb_block[i]);
|
2006-02-01 03:31:06 +01:00
|
|
|
|
2006-02-24 06:40:15 +01:00
|
|
|
spin_lock_irqsave(&ctx_alloc_lock, flags);
|
2006-02-01 03:31:06 +01:00
|
|
|
|
|
|
|
if (CTX_VALID(mm->context)) {
|
|
|
|
unsigned long nr = CTX_NRBITS(mm->context);
|
|
|
|
mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63));
|
|
|
|
}
|
|
|
|
|
2006-02-24 06:40:15 +01:00
|
|
|
spin_unlock_irqrestore(&ctx_alloc_lock, flags);
|
2006-02-01 03:31:06 +01:00
|
|
|
}
|