vmscan: move isolate_lru_page() to vmscan.c

On large memory systems, the VM can spend way too much time scanning
through pages that it cannot (or should not) evict from memory.  Not only
does it use up CPU time, but it also provokes lock contention and can
leave large systems under memory presure in a catatonic state.

This patch series improves VM scalability by:

1) putting filesystem backed, swap backed and unevictable pages
   onto their own LRUs, so the system only scans the pages that it
   can/should evict from memory

2) switching to two handed clock replacement for the anonymous LRUs,
   so the number of pages that need to be scanned when the system
   starts swapping is bound to a reasonable number

3) keeping unevictable pages off the LRU completely, so the
   VM does not waste CPU time scanning them. ramfs, ramdisk,
   SHM_LOCKED shared memory segments and mlock()ed VMA pages
   are keept on the unevictable list.

This patch:

isolate_lru_page logically belongs to be in vmscan.c than migrate.c.

It is tough, because we don't need that function without memory migration
so there is a valid argument to have it in migrate.c.  However a
subsequent patch needs to make use of it in the core mm, so we can happily
move it to vmscan.c.

Also, make the function a little more generic by not requiring that it
adds an isolated page to a given list.  Callers can do that.

	Note that we now have '__isolate_lru_page()', that does
	something quite different, visible outside of vmscan.c
	for use with memory controller.  Methinks we need to
	rationalize these names/purposes.	--lts

[akpm@linux-foundation.org: fix mm/memory_hotplug.c build]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Nick Piggin 2008-10-18 20:26:09 -07:00 committed by Linus Torvalds
parent 71088785c6
commit 62695a84eb
6 changed files with 59 additions and 37 deletions

View File

@ -7,7 +7,6 @@
typedef struct page *new_page_t(struct page *, unsigned long private, int **); typedef struct page *new_page_t(struct page *, unsigned long private, int **);
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
extern int isolate_lru_page(struct page *p, struct list_head *pagelist);
extern int putback_lru_pages(struct list_head *l); extern int putback_lru_pages(struct list_head *l);
extern int migrate_page(struct address_space *, extern int migrate_page(struct address_space *,
struct page *, struct page *); struct page *, struct page *);
@ -21,8 +20,6 @@ extern int migrate_vmas(struct mm_struct *mm,
const nodemask_t *from, const nodemask_t *to, const nodemask_t *from, const nodemask_t *to,
unsigned long flags); unsigned long flags);
#else #else
static inline int isolate_lru_page(struct page *p, struct list_head *list)
{ return -ENOSYS; }
static inline int putback_lru_pages(struct list_head *l) { return 0; } static inline int putback_lru_pages(struct list_head *l) { return 0; }
static inline int migrate_pages(struct list_head *l, new_page_t x, static inline int migrate_pages(struct list_head *l, new_page_t x,
unsigned long private) { return -ENOSYS; } unsigned long private) { return -ENOSYS; }

View File

@ -39,6 +39,8 @@ static inline void __put_page(struct page *page)
atomic_dec(&page->_count); atomic_dec(&page->_count);
} }
extern int isolate_lru_page(struct page *page);
extern void __free_pages_bootmem(struct page *page, unsigned int order); extern void __free_pages_bootmem(struct page *page, unsigned int order);
/* /*

View File

@ -658,8 +658,9 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
* We can skip free pages. And we can only deal with pages on * We can skip free pages. And we can only deal with pages on
* LRU. * LRU.
*/ */
ret = isolate_lru_page(page, &source); ret = isolate_lru_page(page);
if (!ret) { /* Success */ if (!ret) { /* Success */
list_add_tail(&page->lru, &source);
move_pages--; move_pages--;
} else { } else {
/* Becasue we don't have big zone->lock. we should /* Becasue we don't have big zone->lock. we should

View File

@ -93,6 +93,8 @@
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include "internal.h"
/* Internal flags */ /* Internal flags */
#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */ #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
@ -762,8 +764,11 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
/* /*
* Avoid migrating a page that is shared with others. * Avoid migrating a page that is shared with others.
*/ */
if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
isolate_lru_page(page, pagelist); if (!isolate_lru_page(page)) {
list_add_tail(&page->lru, pagelist);
}
}
} }
static struct page *new_node_page(struct page *page, unsigned long node, int **x) static struct page *new_node_page(struct page *page, unsigned long node, int **x)

View File

@ -36,36 +36,6 @@
#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
/*
* Isolate one page from the LRU lists. If successful put it onto
* the indicated list with elevated page count.
*
* Result:
* -EBUSY: page not on LRU list
* 0: page removed from LRU list and added to the specified list.
*/
int isolate_lru_page(struct page *page, struct list_head *pagelist)
{
int ret = -EBUSY;
if (PageLRU(page)) {
struct zone *zone = page_zone(page);
spin_lock_irq(&zone->lru_lock);
if (PageLRU(page) && get_page_unless_zero(page)) {
ret = 0;
ClearPageLRU(page);
if (PageActive(page))
del_page_from_active_list(zone, page);
else
del_page_from_inactive_list(zone, page);
list_add_tail(&page->lru, pagelist);
}
spin_unlock_irq(&zone->lru_lock);
}
return ret;
}
/* /*
* migrate_prep() needs to be called before we start compiling a list of pages * migrate_prep() needs to be called before we start compiling a list of pages
* to be migrated using isolate_lru_page(). * to be migrated using isolate_lru_page().
@ -914,7 +884,9 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
!migrate_all) !migrate_all)
goto put_and_set; goto put_and_set;
err = isolate_lru_page(page, &pagelist); err = isolate_lru_page(page);
if (!err)
list_add_tail(&page->lru, &pagelist);
put_and_set: put_and_set:
/* /*
* Either remove the duplicate refcount from * Either remove the duplicate refcount from

View File

@ -844,6 +844,51 @@ static unsigned long clear_active_flags(struct list_head *page_list)
return nr_active; return nr_active;
} }
/**
* isolate_lru_page - tries to isolate a page from its LRU list
* @page: page to isolate from its LRU list
*
* Isolates a @page from an LRU list, clears PageLRU and adjusts the
* vmstat statistic corresponding to whatever LRU list the page was on.
*
* Returns 0 if the page was removed from an LRU list.
* Returns -EBUSY if the page was not on an LRU list.
*
* The returned page will have PageLRU() cleared. If it was found on
* the active list, it will have PageActive set. That flag may need
* to be cleared by the caller before letting the page go.
*
* The vmstat statistic corresponding to the list on which the page was
* found will be decremented.
*
* Restrictions:
* (1) Must be called with an elevated refcount on the page. This is a
* fundamentnal difference from isolate_lru_pages (which is called
* without a stable reference).
* (2) the lru_lock must not be held.
* (3) interrupts must be enabled.
*/
int isolate_lru_page(struct page *page)
{
int ret = -EBUSY;
if (PageLRU(page)) {
struct zone *zone = page_zone(page);
spin_lock_irq(&zone->lru_lock);
if (PageLRU(page) && get_page_unless_zero(page)) {
ret = 0;
ClearPageLRU(page);
if (PageActive(page))
del_page_from_active_list(zone, page);
else
del_page_from_inactive_list(zone, page);
}
spin_unlock_irq(&zone->lru_lock);
}
return ret;
}
/* /*
* shrink_inactive_list() is a helper for shrink_zone(). It returns the number * shrink_inactive_list() is a helper for shrink_zone(). It returns the number
* of reclaimed pages * of reclaimed pages