vmscan: detect mapped file pages used only once
The VM currently assumes that an inactive, mapped and referenced file page is in use and promotes it to the active list. However, every mapped file page starts out like this and thus a problem arises when workloads create a stream of such pages that are used only for a short time. By flooding the active list with those pages, the VM quickly gets into trouble finding eligible reclaim canditates. The result is long allocation latencies and eviction of the wrong pages. This patch reuses the PG_referenced page flag (used for unmapped file pages) to implement a usage detection that scales with the speed of LRU list cycling (i.e. memory pressure). If the scanner encounters those pages, the flag is set and the page cycled again on the inactive list. Only if it returns with another page table reference it is activated. Otherwise it is reclaimed as 'not recently used cache'. This effectively changes the minimum lifetime of a used-once mapped file page from a full memory cycle to an inactive list cycle, which allows it to occur in linear streams without affecting the stable working set of the system. Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reviewed-by: Rik van Riel <riel@redhat.com> Cc: Minchan Kim <minchan.kim@gmail.com> Cc: OSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Lee Schermerhorn <lee.schermerhorn@hp.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
31c0569c3b
commit
6457474624
|
@ -209,7 +209,7 @@ static inline int page_referenced(struct page *page, int is_locked,
|
||||||
unsigned long *vm_flags)
|
unsigned long *vm_flags)
|
||||||
{
|
{
|
||||||
*vm_flags = 0;
|
*vm_flags = 0;
|
||||||
return TestClearPageReferenced(page);
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
#define try_to_unmap(page, refs) SWAP_FAIL
|
#define try_to_unmap(page, refs) SWAP_FAIL
|
||||||
|
|
|
@ -601,9 +601,6 @@ int page_referenced(struct page *page,
|
||||||
int referenced = 0;
|
int referenced = 0;
|
||||||
int we_locked = 0;
|
int we_locked = 0;
|
||||||
|
|
||||||
if (TestClearPageReferenced(page))
|
|
||||||
referenced++;
|
|
||||||
|
|
||||||
*vm_flags = 0;
|
*vm_flags = 0;
|
||||||
if (page_mapped(page) && page_rmapping(page)) {
|
if (page_mapped(page) && page_rmapping(page)) {
|
||||||
if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
|
if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
|
||||||
|
|
45
mm/vmscan.c
45
mm/vmscan.c
|
@ -561,18 +561,18 @@ redo:
|
||||||
enum page_references {
|
enum page_references {
|
||||||
PAGEREF_RECLAIM,
|
PAGEREF_RECLAIM,
|
||||||
PAGEREF_RECLAIM_CLEAN,
|
PAGEREF_RECLAIM_CLEAN,
|
||||||
|
PAGEREF_KEEP,
|
||||||
PAGEREF_ACTIVATE,
|
PAGEREF_ACTIVATE,
|
||||||
};
|
};
|
||||||
|
|
||||||
static enum page_references page_check_references(struct page *page,
|
static enum page_references page_check_references(struct page *page,
|
||||||
struct scan_control *sc)
|
struct scan_control *sc)
|
||||||
{
|
{
|
||||||
|
int referenced_ptes, referenced_page;
|
||||||
unsigned long vm_flags;
|
unsigned long vm_flags;
|
||||||
int referenced;
|
|
||||||
|
|
||||||
referenced = page_referenced(page, 1, sc->mem_cgroup, &vm_flags);
|
referenced_ptes = page_referenced(page, 1, sc->mem_cgroup, &vm_flags);
|
||||||
if (!referenced)
|
referenced_page = TestClearPageReferenced(page);
|
||||||
return PAGEREF_RECLAIM;
|
|
||||||
|
|
||||||
/* Lumpy reclaim - ignore references */
|
/* Lumpy reclaim - ignore references */
|
||||||
if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
|
if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
|
||||||
|
@ -585,11 +585,36 @@ static enum page_references page_check_references(struct page *page,
|
||||||
if (vm_flags & VM_LOCKED)
|
if (vm_flags & VM_LOCKED)
|
||||||
return PAGEREF_RECLAIM;
|
return PAGEREF_RECLAIM;
|
||||||
|
|
||||||
if (page_mapped(page))
|
if (referenced_ptes) {
|
||||||
return PAGEREF_ACTIVATE;
|
if (PageAnon(page))
|
||||||
|
return PAGEREF_ACTIVATE;
|
||||||
|
/*
|
||||||
|
* All mapped pages start out with page table
|
||||||
|
* references from the instantiating fault, so we need
|
||||||
|
* to look twice if a mapped file page is used more
|
||||||
|
* than once.
|
||||||
|
*
|
||||||
|
* Mark it and spare it for another trip around the
|
||||||
|
* inactive list. Another page table reference will
|
||||||
|
* lead to its activation.
|
||||||
|
*
|
||||||
|
* Note: the mark is set for activated pages as well
|
||||||
|
* so that recently deactivated but used pages are
|
||||||
|
* quickly recovered.
|
||||||
|
*/
|
||||||
|
SetPageReferenced(page);
|
||||||
|
|
||||||
|
if (referenced_page)
|
||||||
|
return PAGEREF_ACTIVATE;
|
||||||
|
|
||||||
|
return PAGEREF_KEEP;
|
||||||
|
}
|
||||||
|
|
||||||
/* Reclaim if clean, defer dirty pages to writeback */
|
/* Reclaim if clean, defer dirty pages to writeback */
|
||||||
return PAGEREF_RECLAIM_CLEAN;
|
if (referenced_page)
|
||||||
|
return PAGEREF_RECLAIM_CLEAN;
|
||||||
|
|
||||||
|
return PAGEREF_RECLAIM;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -657,6 +682,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
|
||||||
switch (references) {
|
switch (references) {
|
||||||
case PAGEREF_ACTIVATE:
|
case PAGEREF_ACTIVATE:
|
||||||
goto activate_locked;
|
goto activate_locked;
|
||||||
|
case PAGEREF_KEEP:
|
||||||
|
goto keep_locked;
|
||||||
case PAGEREF_RECLAIM:
|
case PAGEREF_RECLAIM:
|
||||||
case PAGEREF_RECLAIM_CLEAN:
|
case PAGEREF_RECLAIM_CLEAN:
|
||||||
; /* try to reclaim the page below */
|
; /* try to reclaim the page below */
|
||||||
|
@ -1359,9 +1386,7 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* page_referenced clears PageReferenced */
|
if (page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
|
||||||
if (page_mapped(page) &&
|
|
||||||
page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
|
|
||||||
nr_rotated++;
|
nr_rotated++;
|
||||||
/*
|
/*
|
||||||
* Identify referenced, file-backed active pages and
|
* Identify referenced, file-backed active pages and
|
||||||
|
|
Loading…
Reference in New Issue