sched, numa, mm: Count WS scanning against present PTEs, not virtual memory ranges
By accounting against the present PTEs, scanning speed reflects the actual present (mapped) memory. Suggested-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Signed-off-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: Mel Gorman <mgorman@suse.de>
This commit is contained in:
parent
6e5fb223e8
commit
9f40604cda
|
@ -827,8 +827,8 @@ void task_numa_work(struct callback_head *work)
|
||||||
struct task_struct *p = current;
|
struct task_struct *p = current;
|
||||||
struct mm_struct *mm = p->mm;
|
struct mm_struct *mm = p->mm;
|
||||||
struct vm_area_struct *vma;
|
struct vm_area_struct *vma;
|
||||||
unsigned long offset, end;
|
unsigned long start, end;
|
||||||
long length;
|
long pages;
|
||||||
|
|
||||||
WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
|
WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work));
|
||||||
|
|
||||||
|
@ -858,18 +858,20 @@ void task_numa_work(struct callback_head *work)
|
||||||
if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
|
if (cmpxchg(&mm->numa_next_scan, migrate, next_scan) != migrate)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
offset = mm->numa_scan_offset;
|
start = mm->numa_scan_offset;
|
||||||
length = sysctl_numa_balancing_scan_size;
|
pages = sysctl_numa_balancing_scan_size;
|
||||||
length <<= 20;
|
pages <<= 20 - PAGE_SHIFT; /* MB in pages */
|
||||||
|
if (!pages)
|
||||||
|
return;
|
||||||
|
|
||||||
down_read(&mm->mmap_sem);
|
down_read(&mm->mmap_sem);
|
||||||
vma = find_vma(mm, offset);
|
vma = find_vma(mm, start);
|
||||||
if (!vma) {
|
if (!vma) {
|
||||||
reset_ptenuma_scan(p);
|
reset_ptenuma_scan(p);
|
||||||
offset = 0;
|
start = 0;
|
||||||
vma = mm->mmap;
|
vma = mm->mmap;
|
||||||
}
|
}
|
||||||
for (; vma && length > 0; vma = vma->vm_next) {
|
for (; vma; vma = vma->vm_next) {
|
||||||
if (!vma_migratable(vma))
|
if (!vma_migratable(vma))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
|
@ -877,15 +879,19 @@ void task_numa_work(struct callback_head *work)
|
||||||
if (((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) < HPAGE_PMD_NR)
|
if (((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) < HPAGE_PMD_NR)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
offset = max(offset, vma->vm_start);
|
do {
|
||||||
end = min(ALIGN(offset + length, HPAGE_SIZE), vma->vm_end);
|
start = max(start, vma->vm_start);
|
||||||
length -= end - offset;
|
end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
|
||||||
|
end = min(end, vma->vm_end);
|
||||||
|
pages -= change_prot_numa(vma, start, end);
|
||||||
|
|
||||||
change_prot_numa(vma, offset, end);
|
start = end;
|
||||||
|
if (pages <= 0)
|
||||||
offset = end;
|
goto out;
|
||||||
|
} while (end != vma->vm_end);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
out:
|
||||||
/*
|
/*
|
||||||
* It is possible to reach the end of the VMA list but the last few VMAs are
|
* It is possible to reach the end of the VMA list but the last few VMAs are
|
||||||
* not guaranteed to the vma_migratable. If they are not, we would find the
|
* not guaranteed to the vma_migratable. If they are not, we would find the
|
||||||
|
@ -893,7 +899,7 @@ void task_numa_work(struct callback_head *work)
|
||||||
* so check it now.
|
* so check it now.
|
||||||
*/
|
*/
|
||||||
if (vma)
|
if (vma)
|
||||||
mm->numa_scan_offset = offset;
|
mm->numa_scan_offset = start;
|
||||||
else
|
else
|
||||||
reset_ptenuma_scan(p);
|
reset_ptenuma_scan(p);
|
||||||
up_read(&mm->mmap_sem);
|
up_read(&mm->mmap_sem);
|
||||||
|
|
Loading…
Reference in New Issue