2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* linux/mm/mincore.c
|
|
|
|
*
|
2006-12-16 18:44:32 +01:00
|
|
|
* Copyright (C) 1994-2006 Linus Torvalds
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The mincore() system call.
|
|
|
|
*/
|
|
|
|
#include <linux/pagemap.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 09:04:11 +01:00
|
|
|
#include <linux/gfp.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/mman.h>
|
|
|
|
#include <linux/syscalls.h>
|
2007-02-12 09:51:39 +01:00
|
|
|
#include <linux/swap.h>
|
|
|
|
#include <linux/swapops.h>
|
2009-12-15 02:59:58 +01:00
|
|
|
#include <linux/hugetlb.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2016-12-24 20:46:01 +01:00
|
|
|
#include <linux/uaccess.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <asm/pgtable.h>
|
|
|
|
|
2015-02-12 00:28:11 +01:00
|
|
|
static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr,
|
|
|
|
unsigned long end, struct mm_walk *walk)
|
2010-05-24 23:32:10 +02:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
2015-02-12 00:28:11 +01:00
|
|
|
unsigned char present;
|
|
|
|
unsigned char *vec = walk->private;
|
2010-05-24 23:32:10 +02:00
|
|
|
|
2015-02-12 00:28:11 +01:00
|
|
|
/*
|
|
|
|
* Hugepages under user process are always in RAM and never
|
|
|
|
* swapped out, but theoretically it needs to be checked.
|
|
|
|
*/
|
|
|
|
present = pte && !huge_pte_none(huge_ptep_get(pte));
|
|
|
|
for (; addr != end; vec++, addr += PAGE_SIZE)
|
|
|
|
*vec = present;
|
|
|
|
walk->private = vec;
|
2010-05-24 23:32:10 +02:00
|
|
|
#else
|
|
|
|
BUG();
|
|
|
|
#endif
|
2015-02-12 00:28:11 +01:00
|
|
|
return 0;
|
2010-05-24 23:32:10 +02:00
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* Later we can get more picky about what "in core" means precisely.
|
|
|
|
* For now, simply check to see if the page is in the page cache,
|
|
|
|
* and is up to date; i.e. that no page-in operation would be required
|
|
|
|
* at this time if an application were to map and access this page.
|
|
|
|
*/
|
2007-02-12 09:51:39 +01:00
|
|
|
static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
unsigned char present = 0;
|
2007-02-12 09:51:39 +01:00
|
|
|
struct page *page;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-02-12 09:51:39 +01:00
|
|
|
/*
|
|
|
|
* When tmpfs swaps out a page from a file, any process mapping that
|
|
|
|
* file will not get a swp_entry_t in its pte, but rather it is like
|
|
|
|
* any other file mapping (ie. marked !present and faulted in with
|
2008-04-28 11:12:10 +02:00
|
|
|
* tmpfs's .fault). So swapped out tmpfs mappings are tested here.
|
2007-02-12 09:51:39 +01:00
|
|
|
*/
|
2011-08-04 01:21:27 +02:00
|
|
|
#ifdef CONFIG_SWAP
|
2014-04-03 23:47:46 +02:00
|
|
|
if (shmem_mapping(mapping)) {
|
|
|
|
page = find_get_entry(mapping, pgoff);
|
|
|
|
/*
|
|
|
|
* shmem/tmpfs may return swap: account for swapcache
|
|
|
|
* page too.
|
|
|
|
*/
|
|
|
|
if (radix_tree_exceptional_entry(page)) {
|
|
|
|
swp_entry_t swp = radix_to_swp_entry(page);
|
mm, swap: use offset of swap entry as key of swap cache
This patch is to improve the performance of swap cache operations when
the type of the swap device is not 0. Originally, the whole swap entry
value is used as the key of the swap cache, even though there is one
radix tree for each swap device. If the type of the swap device is not
0, the height of the radix tree of the swap cache will be increased
unnecessary, especially on 64bit architecture. For example, for a 1GB
swap device on the x86_64 architecture, the height of the radix tree of
the swap cache is 11. But if the offset of the swap entry is used as
the key of the swap cache, the height of the radix tree of the swap
cache is 4. The increased height causes unnecessary radix tree
descending and increased cache footprint.
This patch reduces the height of the radix tree of the swap cache via
using the offset of the swap entry instead of the whole swap entry value
as the key of the swap cache. In 32 processes sequential swap out test
case on a Xeon E5 v3 system with RAM disk as swap, the lock contention
for the spinlock of the swap cache is reduced from 20.15% to 12.19%,
when the type of the swap device is 1.
Use the whole swap entry as key,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irq.__add_to_swap_cache.add_to_swap_cache.add_to_swap.shrink_page_list: 10.37,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irqsave.__remove_mapping.shrink_page_list.shrink_inactive_list.shrink_node_memcg: 9.78,
Use the swap offset as key,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irq.__add_to_swap_cache.add_to_swap_cache.add_to_swap.shrink_page_list: 6.25,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irqsave.__remove_mapping.shrink_page_list.shrink_inactive_list.shrink_node_memcg: 5.94,
Link: http://lkml.kernel.org/r/1473270649-27229-1-git-send-email-ying.huang@intel.com
Signed-off-by: "Huang, Ying" <ying.huang@intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-10-08 02:00:21 +02:00
|
|
|
page = find_get_page(swap_address_space(swp),
|
|
|
|
swp_offset(swp));
|
2014-04-03 23:47:46 +02:00
|
|
|
}
|
|
|
|
} else
|
|
|
|
page = find_get_page(mapping, pgoff);
|
|
|
|
#else
|
|
|
|
page = find_get_page(mapping, pgoff);
|
2011-08-04 01:21:27 +02:00
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
if (page) {
|
|
|
|
present = PageUptodate(page);
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 14:29:47 +02:00
|
|
|
put_page(page);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return present;
|
|
|
|
}
|
|
|
|
|
2015-02-12 00:28:11 +01:00
|
|
|
static int __mincore_unmapped_range(unsigned long addr, unsigned long end,
|
|
|
|
struct vm_area_struct *vma, unsigned char *vec)
|
2010-05-24 23:32:10 +02:00
|
|
|
{
|
2010-05-24 23:32:11 +02:00
|
|
|
unsigned long nr = (end - addr) >> PAGE_SHIFT;
|
2010-05-24 23:32:10 +02:00
|
|
|
int i;
|
|
|
|
|
|
|
|
if (vma->vm_file) {
|
|
|
|
pgoff_t pgoff;
|
|
|
|
|
|
|
|
pgoff = linear_page_index(vma, addr);
|
|
|
|
for (i = 0; i < nr; i++, pgoff++)
|
|
|
|
vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff);
|
|
|
|
} else {
|
|
|
|
for (i = 0; i < nr; i++)
|
|
|
|
vec[i] = 0;
|
|
|
|
}
|
2015-02-12 00:28:11 +01:00
|
|
|
return nr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int mincore_unmapped_range(unsigned long addr, unsigned long end,
|
|
|
|
struct mm_walk *walk)
|
|
|
|
{
|
|
|
|
walk->private += __mincore_unmapped_range(addr, end,
|
|
|
|
walk->vma, walk->private);
|
|
|
|
return 0;
|
2010-05-24 23:32:10 +02:00
|
|
|
}
|
|
|
|
|
2015-02-12 00:28:11 +01:00
|
|
|
static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
|
|
|
|
struct mm_walk *walk)
|
2010-05-24 23:32:10 +02:00
|
|
|
{
|
|
|
|
spinlock_t *ptl;
|
2015-02-12 00:28:11 +01:00
|
|
|
struct vm_area_struct *vma = walk->vma;
|
2010-05-24 23:32:10 +02:00
|
|
|
pte_t *ptep;
|
2015-02-12 00:28:11 +01:00
|
|
|
unsigned char *vec = walk->private;
|
|
|
|
int nr = (end - addr) >> PAGE_SHIFT;
|
|
|
|
|
2016-01-22 01:40:25 +01:00
|
|
|
ptl = pmd_trans_huge_lock(pmd, vma);
|
|
|
|
if (ptl) {
|
2015-02-12 00:28:11 +01:00
|
|
|
memset(vec, 1, nr);
|
|
|
|
spin_unlock(ptl);
|
|
|
|
goto out;
|
|
|
|
}
|
2010-05-24 23:32:10 +02:00
|
|
|
|
2015-02-12 00:28:11 +01:00
|
|
|
if (pmd_trans_unstable(pmd)) {
|
|
|
|
__mincore_unmapped_range(addr, end, vma, vec);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
ptep = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
|
|
|
|
for (; addr != end; ptep++, addr += PAGE_SIZE) {
|
2010-05-24 23:32:10 +02:00
|
|
|
pte_t pte = *ptep;
|
|
|
|
|
|
|
|
if (pte_none(pte))
|
2015-02-12 00:28:11 +01:00
|
|
|
__mincore_unmapped_range(addr, addr + PAGE_SIZE,
|
|
|
|
vma, vec);
|
2010-05-24 23:32:10 +02:00
|
|
|
else if (pte_present(pte))
|
2010-05-24 23:32:11 +02:00
|
|
|
*vec = 1;
|
2015-02-10 23:10:04 +01:00
|
|
|
else { /* pte is a swap entry */
|
2010-05-24 23:32:10 +02:00
|
|
|
swp_entry_t entry = pte_to_swp_entry(pte);
|
|
|
|
|
2014-12-13 01:55:07 +01:00
|
|
|
if (non_swap_entry(entry)) {
|
|
|
|
/*
|
|
|
|
* migration or hwpoison entries are always
|
|
|
|
* uptodate
|
|
|
|
*/
|
2010-05-24 23:32:11 +02:00
|
|
|
*vec = 1;
|
2010-05-24 23:32:10 +02:00
|
|
|
} else {
|
|
|
|
#ifdef CONFIG_SWAP
|
2013-02-23 01:34:37 +01:00
|
|
|
*vec = mincore_page(swap_address_space(entry),
|
mm, swap: use offset of swap entry as key of swap cache
This patch is to improve the performance of swap cache operations when
the type of the swap device is not 0. Originally, the whole swap entry
value is used as the key of the swap cache, even though there is one
radix tree for each swap device. If the type of the swap device is not
0, the height of the radix tree of the swap cache will be increased
unnecessary, especially on 64bit architecture. For example, for a 1GB
swap device on the x86_64 architecture, the height of the radix tree of
the swap cache is 11. But if the offset of the swap entry is used as
the key of the swap cache, the height of the radix tree of the swap
cache is 4. The increased height causes unnecessary radix tree
descending and increased cache footprint.
This patch reduces the height of the radix tree of the swap cache via
using the offset of the swap entry instead of the whole swap entry value
as the key of the swap cache. In 32 processes sequential swap out test
case on a Xeon E5 v3 system with RAM disk as swap, the lock contention
for the spinlock of the swap cache is reduced from 20.15% to 12.19%,
when the type of the swap device is 1.
Use the whole swap entry as key,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irq.__add_to_swap_cache.add_to_swap_cache.add_to_swap.shrink_page_list: 10.37,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irqsave.__remove_mapping.shrink_page_list.shrink_inactive_list.shrink_node_memcg: 9.78,
Use the swap offset as key,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irq.__add_to_swap_cache.add_to_swap_cache.add_to_swap.shrink_page_list: 6.25,
perf-profile.calltrace.cycles-pp._raw_spin_lock_irqsave.__remove_mapping.shrink_page_list.shrink_inactive_list.shrink_node_memcg: 5.94,
Link: http://lkml.kernel.org/r/1473270649-27229-1-git-send-email-ying.huang@intel.com
Signed-off-by: "Huang, Ying" <ying.huang@intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Aaron Lu <aaron.lu@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-10-08 02:00:21 +02:00
|
|
|
swp_offset(entry));
|
2010-05-24 23:32:10 +02:00
|
|
|
#else
|
|
|
|
WARN_ON(1);
|
2010-05-24 23:32:11 +02:00
|
|
|
*vec = 1;
|
2010-05-24 23:32:10 +02:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
2010-05-24 23:32:11 +02:00
|
|
|
vec++;
|
2015-02-12 00:28:11 +01:00
|
|
|
}
|
2010-05-24 23:32:10 +02:00
|
|
|
pte_unmap_unlock(ptep - 1, ptl);
|
2015-02-12 00:28:11 +01:00
|
|
|
out:
|
|
|
|
walk->private += nr;
|
|
|
|
cond_resched();
|
|
|
|
return 0;
|
2010-05-24 23:32:11 +02:00
|
|
|
}
|
|
|
|
|
2006-12-16 18:44:32 +01:00
|
|
|
/*
|
|
|
|
* Do a chunk of "sys_mincore()". We've already checked
|
|
|
|
* all the arguments, we hold the mmap semaphore: we should
|
|
|
|
* just return the amount of info we're asked for.
|
|
|
|
*/
|
2010-05-24 23:32:09 +02:00
|
|
|
static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-05-24 23:32:09 +02:00
|
|
|
struct vm_area_struct *vma;
|
2010-05-24 23:32:11 +02:00
|
|
|
unsigned long end;
|
2015-02-12 00:28:11 +01:00
|
|
|
int err;
|
|
|
|
struct mm_walk mincore_walk = {
|
|
|
|
.pmd_entry = mincore_pte_range,
|
|
|
|
.pte_hole = mincore_unmapped_range,
|
|
|
|
.hugetlb_entry = mincore_hugetlb,
|
|
|
|
.private = vec,
|
|
|
|
};
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-05-24 23:32:09 +02:00
|
|
|
vma = find_vma(current->mm, addr);
|
2006-12-17 01:01:50 +01:00
|
|
|
if (!vma || addr < vma->vm_start)
|
|
|
|
return -ENOMEM;
|
2015-02-12 00:28:11 +01:00
|
|
|
mincore_walk.mm = vma->vm_mm;
|
2010-05-24 23:32:11 +02:00
|
|
|
end = min(vma->vm_end, addr + (pages << PAGE_SHIFT));
|
2015-02-12 00:28:11 +01:00
|
|
|
err = walk_page_range(addr, end, &mincore_walk);
|
|
|
|
if (err < 0)
|
|
|
|
return err;
|
2010-05-24 23:32:11 +02:00
|
|
|
return (end - addr) >> PAGE_SHIFT;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The mincore(2) system call.
|
|
|
|
*
|
|
|
|
* mincore() returns the memory residency status of the pages in the
|
|
|
|
* current process's address space specified by [addr, addr + len).
|
|
|
|
* The status is returned in a vector of bytes. The least significant
|
|
|
|
* bit of each byte is 1 if the referenced page is in memory, otherwise
|
|
|
|
* it is zero.
|
|
|
|
*
|
|
|
|
* Because the status of a page can change after mincore() checks it
|
|
|
|
* but before it returns to the application, the returned vector may
|
|
|
|
* contain stale information. Only locked pages are guaranteed to
|
|
|
|
* remain in memory.
|
|
|
|
*
|
|
|
|
* return values:
|
|
|
|
* zero - success
|
|
|
|
* -EFAULT - vec points to an illegal address
|
2016-04-01 14:29:48 +02:00
|
|
|
* -EINVAL - addr is not a multiple of PAGE_SIZE
|
2005-04-17 00:20:36 +02:00
|
|
|
* -ENOMEM - Addresses in the range [addr, addr + len] are
|
|
|
|
* invalid for the address space of this process, or
|
|
|
|
* specify one or more pages which are not currently
|
|
|
|
* mapped
|
|
|
|
* -EAGAIN - A kernel resource was temporarily unavailable.
|
|
|
|
*/
|
2009-01-14 14:14:16 +01:00
|
|
|
SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len,
|
|
|
|
unsigned char __user *, vec)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-12-16 18:44:32 +01:00
|
|
|
long retval;
|
|
|
|
unsigned long pages;
|
|
|
|
unsigned char *tmp;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-12-16 18:44:32 +01:00
|
|
|
/* Check the start address: needs to be page-aligned.. */
|
mm, fs: get rid of PAGE_CACHE_* and page_cache_{get,release} macros
PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} macros were introduced *long* time
ago with promise that one day it will be possible to implement page
cache with bigger chunks than PAGE_SIZE.
This promise never materialized. And unlikely will.
We have many places where PAGE_CACHE_SIZE assumed to be equal to
PAGE_SIZE. And it's constant source of confusion on whether
PAGE_CACHE_* or PAGE_* constant should be used in a particular case,
especially on the border between fs and mm.
Global switching to PAGE_CACHE_SIZE != PAGE_SIZE would cause to much
breakage to be doable.
Let's stop pretending that pages in page cache are special. They are
not.
The changes are pretty straight-forward:
- <foo> << (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- <foo> >> (PAGE_CACHE_SHIFT - PAGE_SHIFT) -> <foo>;
- PAGE_CACHE_{SIZE,SHIFT,MASK,ALIGN} -> PAGE_{SIZE,SHIFT,MASK,ALIGN};
- page_cache_get() -> get_page();
- page_cache_release() -> put_page();
This patch contains automated changes generated with coccinelle using
script below. For some reason, coccinelle doesn't patch header files.
I've called spatch for them manually.
The only adjustment after coccinelle is revert of changes to
PAGE_CAHCE_ALIGN definition: we are going to drop it later.
There are few places in the code where coccinelle didn't reach. I'll
fix them manually in a separate patch. Comments and documentation also
will be addressed with the separate patch.
virtual patch
@@
expression E;
@@
- E << (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
expression E;
@@
- E >> (PAGE_CACHE_SHIFT - PAGE_SHIFT)
+ E
@@
@@
- PAGE_CACHE_SHIFT
+ PAGE_SHIFT
@@
@@
- PAGE_CACHE_SIZE
+ PAGE_SIZE
@@
@@
- PAGE_CACHE_MASK
+ PAGE_MASK
@@
expression E;
@@
- PAGE_CACHE_ALIGN(E)
+ PAGE_ALIGN(E)
@@
expression E;
@@
- page_cache_get(E)
+ get_page(E)
@@
expression E;
@@
- page_cache_release(E)
+ put_page(E)
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-04-01 14:29:47 +02:00
|
|
|
if (start & ~PAGE_MASK)
|
2006-12-16 18:44:32 +01:00
|
|
|
return -EINVAL;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-12-16 18:44:32 +01:00
|
|
|
/* ..and we need to be passed a valid user-space range */
|
|
|
|
if (!access_ok(VERIFY_READ, (void __user *) start, len))
|
|
|
|
return -ENOMEM;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2016-04-01 14:29:48 +02:00
|
|
|
/* This also avoids any overflows on PAGE_ALIGN */
|
2006-12-16 18:44:32 +01:00
|
|
|
pages = len >> PAGE_SHIFT;
|
2015-11-06 03:46:38 +01:00
|
|
|
pages += (offset_in_page(len)) != 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-12-16 18:44:32 +01:00
|
|
|
if (!access_ok(VERIFY_WRITE, vec, pages))
|
|
|
|
return -EFAULT;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-12-16 18:44:32 +01:00
|
|
|
tmp = (void *) __get_free_page(GFP_USER);
|
|
|
|
if (!tmp)
|
2006-12-17 01:01:50 +01:00
|
|
|
return -EAGAIN;
|
2006-12-16 18:44:32 +01:00
|
|
|
|
|
|
|
retval = 0;
|
|
|
|
while (pages) {
|
|
|
|
/*
|
|
|
|
* Do at most PAGE_SIZE entries per iteration, due to
|
|
|
|
* the temporary buffer size.
|
|
|
|
*/
|
|
|
|
down_read(¤t->mm->mmap_sem);
|
2010-05-24 23:32:09 +02:00
|
|
|
retval = do_mincore(start, min(pages, PAGE_SIZE), tmp);
|
2006-12-16 18:44:32 +01:00
|
|
|
up_read(¤t->mm->mmap_sem);
|
|
|
|
|
|
|
|
if (retval <= 0)
|
|
|
|
break;
|
|
|
|
if (copy_to_user(vec, tmp, retval)) {
|
|
|
|
retval = -EFAULT;
|
|
|
|
break;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-12-16 18:44:32 +01:00
|
|
|
pages -= retval;
|
|
|
|
vec += retval;
|
|
|
|
start += retval << PAGE_SHIFT;
|
|
|
|
retval = 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-12-16 18:44:32 +01:00
|
|
|
free_page((unsigned long) tmp);
|
|
|
|
return retval;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|