2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* Resizable virtual memory filesystem for Linux.
|
|
|
|
*
|
|
|
|
* Copyright (C) 2000 Linus Torvalds.
|
|
|
|
* 2000 Transmeta Corp.
|
|
|
|
* 2000-2001 Christoph Rohland
|
|
|
|
* 2000-2001 SAP AG
|
|
|
|
* 2002 Red Hat Inc.
|
2005-06-22 02:15:04 +02:00
|
|
|
* Copyright (C) 2002-2005 Hugh Dickins.
|
|
|
|
* Copyright (C) 2002-2005 VERITAS Software Corporation.
|
2005-04-17 00:20:36 +02:00
|
|
|
* Copyright (C) 2004 Andi Kleen, SuSE Labs
|
|
|
|
*
|
|
|
|
* Extended attribute support for tmpfs:
|
|
|
|
* Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
|
|
|
|
* Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
|
|
|
|
*
|
2009-01-06 23:40:20 +01:00
|
|
|
* tiny-shmem:
|
|
|
|
* Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
|
|
|
|
*
|
2005-04-17 00:20:36 +02:00
|
|
|
* This file is released under the GPL.
|
|
|
|
*/
|
|
|
|
|
2009-01-06 23:40:20 +01:00
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/vfs.h>
|
|
|
|
#include <linux/mount.h>
|
2009-04-13 23:40:12 +02:00
|
|
|
#include <linux/pagemap.h>
|
2009-01-06 23:40:20 +01:00
|
|
|
#include <linux/file.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/swap.h>
|
2009-02-11 05:01:19 +01:00
|
|
|
#include <linux/ima.h>
|
2009-01-06 23:40:20 +01:00
|
|
|
|
|
|
|
static struct vfsmount *shm_mnt;
|
|
|
|
|
|
|
|
#ifdef CONFIG_SHMEM
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* This virtual memory filesystem is heavily based on the ramfs. It
|
|
|
|
* extends ramfs by the ability to use swap and honor resource limits
|
|
|
|
* which makes it a completely usable filesystem.
|
|
|
|
*/
|
|
|
|
|
2006-09-29 11:01:35 +02:00
|
|
|
#include <linux/xattr.h>
|
2007-07-17 13:04:28 +02:00
|
|
|
#include <linux/exportfs.h>
|
2006-09-29 11:01:35 +02:00
|
|
|
#include <linux/generic_acl.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <linux/mman.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/backing-dev.h>
|
|
|
|
#include <linux/shmem_fs.h>
|
|
|
|
#include <linux/writeback.h>
|
|
|
|
#include <linux/vfs.h>
|
|
|
|
#include <linux/blkdev.h>
|
|
|
|
#include <linux/security.h>
|
|
|
|
#include <linux/swapops.h>
|
|
|
|
#include <linux/mempolicy.h>
|
|
|
|
#include <linux/namei.h>
|
2006-02-22 00:49:47 +01:00
|
|
|
#include <linux/ctype.h>
|
[PATCH] add migratepage address space op to shmem
Basic problem: pages of a shared memory segment can only be migrated once.
In 2.6.16 through 2.6.17-rc1, shared memory mappings do not have a
migratepage address space op. Therefore, migrate_pages() falls back to
default processing. In this path, it will try to pageout() dirty pages.
Once a shared memory page has been migrated it becomes dirty, so
migrate_pages() will try to page it out. However, because the page count
is 3 [cache + current + pte], pageout() will return PAGE_KEEP because
is_page_cache_freeable() returns false. This will abort all subsequent
migrations.
This patch adds a migratepage address space op to shared memory segments to
avoid taking the default path. We use the "migrate_page()" function
because it knows how to migrate dirty pages. This allows shared memory
segment pages to migrate, subject to other conditions such as # pte's
referencing the page [page_mapcount(page)], when requested.
I think this is safe. If we're migrating a shared memory page, then we
found the page via a page table, so it must be in memory.
Can be verified with memtoy and the shmem-mbind-test script, both
available at: http://free.linux.hp.com/~lts/Tools/
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-22 11:35:48 +02:00
|
|
|
#include <linux/migrate.h>
|
2006-09-26 08:31:11 +02:00
|
|
|
#include <linux/highmem.h>
|
2008-02-08 13:21:48 +01:00
|
|
|
#include <linux/seq_file.h>
|
2008-10-07 20:00:12 +02:00
|
|
|
#include <linux/magic.h>
|
[PATCH] add migratepage address space op to shmem
Basic problem: pages of a shared memory segment can only be migrated once.
In 2.6.16 through 2.6.17-rc1, shared memory mappings do not have a
migratepage address space op. Therefore, migrate_pages() falls back to
default processing. In this path, it will try to pageout() dirty pages.
Once a shared memory page has been migrated it becomes dirty, so
migrate_pages() will try to page it out. However, because the page count
is 3 [cache + current + pte], pageout() will return PAGE_KEEP because
is_page_cache_freeable() returns false. This will abort all subsequent
migrations.
This patch adds a migratepage address space op to shared memory segments to
avoid taking the default path. We use the "migrate_page()" function
because it knows how to migrate dirty pages. This allows shared memory
segment pages to migrate, subject to other conditions such as # pte's
referencing the page [page_mapcount(page)], when requested.
I think this is safe. If we're migrating a shared memory page, then we
found the page via a page table, so it must be in memory.
Can be verified with memtoy and the shmem-mbind-test script, both
available at: http://free.linux.hp.com/~lts/Tools/
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-22 11:35:48 +02:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
#include <asm/uaccess.h>
|
|
|
|
#include <asm/div64.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
|
2009-04-13 23:40:12 +02:00
|
|
|
/*
|
|
|
|
* The maximum size of a shmem/tmpfs file is limited by the maximum size of
|
|
|
|
* its triple-indirect swap vector - see illustration at shmem_swp_entry().
|
|
|
|
*
|
|
|
|
* With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel,
|
|
|
|
* but one eighth of that on a 64-bit kernel. With 8kB page size, maximum
|
|
|
|
* file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel,
|
|
|
|
* MAX_LFS_FILESIZE being then more restrictive than swap vector layout.
|
|
|
|
*
|
|
|
|
* We use / and * instead of shifts in the definitions below, so that the swap
|
|
|
|
* vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE.
|
|
|
|
*/
|
2005-04-17 00:20:36 +02:00
|
|
|
#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
|
2009-04-13 23:40:11 +02:00
|
|
|
#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2009-04-13 23:40:12 +02:00
|
|
|
#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
|
|
|
|
#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2009-04-13 23:40:12 +02:00
|
|
|
#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
|
|
|
|
#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
|
|
|
|
|
|
|
|
#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
|
2005-04-17 00:20:36 +02:00
|
|
|
#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
|
|
|
|
|
|
|
|
/* info->flags needs VM_flags to handle pagein/truncate races efficiently */
|
|
|
|
#define SHMEM_PAGEIN VM_READ
|
|
|
|
#define SHMEM_TRUNCATE VM_WRITE
|
|
|
|
|
|
|
|
/* Definition to limit shmem_truncate's steps between cond_rescheds */
|
|
|
|
#define LATENCY_LIMIT 64
|
|
|
|
|
|
|
|
/* Pretend that each entry is of this size in directory's i_size */
|
|
|
|
#define BOGO_DIRENT_SIZE 20
|
|
|
|
|
|
|
|
/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
|
|
|
|
enum sgp_type {
|
|
|
|
SGP_READ, /* don't exceed i_size, don't allocate page */
|
|
|
|
SGP_CACHE, /* don't exceed i_size, may allocate page */
|
2008-02-05 07:28:51 +01:00
|
|
|
SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
|
2005-04-17 00:20:36 +02:00
|
|
|
SGP_WRITE, /* may exceed i_size, may allocate page */
|
|
|
|
};
|
|
|
|
|
2008-02-08 13:21:49 +01:00
|
|
|
#ifdef CONFIG_TMPFS
|
2008-02-08 13:21:48 +01:00
|
|
|
static unsigned long shmem_default_max_blocks(void)
|
|
|
|
{
|
|
|
|
return totalram_pages / 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long shmem_default_max_inodes(void)
|
|
|
|
{
|
|
|
|
return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
|
|
|
|
}
|
2008-02-08 13:21:49 +01:00
|
|
|
#endif
|
2008-02-08 13:21:48 +01:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|
|
|
struct page **pagep, enum sgp_type sgp, int *type);
|
|
|
|
|
2005-10-21 09:18:50 +02:00
|
|
|
static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The above definition of ENTRIES_PER_PAGE, and the use of
|
|
|
|
* BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
|
|
|
|
* might be reconsidered if it ever diverges from PAGE_SIZE.
|
2007-07-17 13:03:05 +02:00
|
|
|
*
|
2007-10-16 10:25:52 +02:00
|
|
|
* Mobility flags are masked out as swap vectors cannot move
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2007-10-16 10:25:52 +02:00
|
|
|
return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
|
2007-07-17 13:03:05 +02:00
|
|
|
PAGE_CACHE_SHIFT-PAGE_SHIFT);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void shmem_dir_free(struct page *page)
|
|
|
|
{
|
|
|
|
__free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct page **shmem_dir_map(struct page *page)
|
|
|
|
{
|
|
|
|
return (struct page **)kmap_atomic(page, KM_USER0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void shmem_dir_unmap(struct page **dir)
|
|
|
|
{
|
|
|
|
kunmap_atomic(dir, KM_USER0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static swp_entry_t *shmem_swp_map(struct page *page)
|
|
|
|
{
|
|
|
|
return (swp_entry_t *)kmap_atomic(page, KM_USER1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void shmem_swp_balance_unmap(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* When passing a pointer to an i_direct entry, to code which
|
|
|
|
* also handles indirect entries and so will shmem_swp_unmap,
|
|
|
|
* we must arrange for the preempt count to remain in balance.
|
|
|
|
* What kmap_atomic of a lowmem page does depends on config
|
|
|
|
* and architecture, so pretend to kmap_atomic some lowmem page.
|
|
|
|
*/
|
|
|
|
(void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void shmem_swp_unmap(swp_entry_t *entry)
|
|
|
|
{
|
|
|
|
kunmap_atomic(entry, KM_USER1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
|
|
|
|
{
|
|
|
|
return sb->s_fs_info;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* shmem_file_setup pre-accounts the whole fixed size of a VM object,
|
|
|
|
* for shared memory and for shared anonymous (/dev/zero) mappings
|
|
|
|
* (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
|
|
|
|
* consistent with the pre-accounting of private mappings ...
|
|
|
|
*/
|
|
|
|
static inline int shmem_acct_size(unsigned long flags, loff_t size)
|
|
|
|
{
|
2009-02-24 21:51:52 +01:00
|
|
|
return (flags & VM_NORESERVE) ?
|
|
|
|
0 : security_vm_enough_memory_kern(VM_ACCT(size));
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void shmem_unacct_size(unsigned long flags, loff_t size)
|
|
|
|
{
|
2009-02-24 21:51:52 +01:00
|
|
|
if (!(flags & VM_NORESERVE))
|
2005-04-17 00:20:36 +02:00
|
|
|
vm_unacct_memory(VM_ACCT(size));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* ... whereas tmpfs objects are accounted incrementally as
|
|
|
|
* pages are allocated, in order to allow huge sparse files.
|
|
|
|
* shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
|
|
|
|
* so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
|
|
|
|
*/
|
|
|
|
static inline int shmem_acct_block(unsigned long flags)
|
|
|
|
{
|
2009-02-24 21:51:52 +01:00
|
|
|
return (flags & VM_NORESERVE) ?
|
|
|
|
security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void shmem_unacct_blocks(unsigned long flags, long pages)
|
|
|
|
{
|
2009-02-24 21:51:52 +01:00
|
|
|
if (flags & VM_NORESERVE)
|
2005-04-17 00:20:36 +02:00
|
|
|
vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
|
|
|
|
}
|
|
|
|
|
2007-03-05 09:30:28 +01:00
|
|
|
static const struct super_operations shmem_ops;
|
2006-06-28 13:26:44 +02:00
|
|
|
static const struct address_space_operations shmem_aops;
|
2006-12-07 05:40:36 +01:00
|
|
|
static const struct file_operations shmem_file_operations;
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_inode_operations;
|
|
|
|
static const struct inode_operations shmem_dir_inode_operations;
|
|
|
|
static const struct inode_operations shmem_special_inode_operations;
|
2005-04-17 00:20:36 +02:00
|
|
|
static struct vm_operations_struct shmem_vm_ops;
|
|
|
|
|
2005-09-07 00:17:45 +02:00
|
|
|
static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.ra_pages = 0, /* No readahead */
|
2008-10-19 05:26:32 +02:00
|
|
|
.capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
|
2005-04-17 00:20:36 +02:00
|
|
|
.unplug_io_fn = default_unplug_io_fn,
|
|
|
|
};
|
|
|
|
|
|
|
|
static LIST_HEAD(shmem_swaplist);
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
static DEFINE_MUTEX(shmem_swaplist_mutex);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
static void shmem_free_blocks(struct inode *inode, long pages)
|
|
|
|
{
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
|
2005-06-22 02:15:04 +02:00
|
|
|
if (sbinfo->max_blocks) {
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
sbinfo->free_blocks += pages;
|
|
|
|
inode->i_blocks -= pages*BLOCKS_PER_PAGE;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-02-05 07:28:47 +01:00
|
|
|
static int shmem_reserve_inode(struct super_block *sb)
|
|
|
|
{
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
|
|
|
|
if (sbinfo->max_inodes) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (!sbinfo->free_inodes) {
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
return -ENOSPC;
|
|
|
|
}
|
|
|
|
sbinfo->free_inodes--;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void shmem_free_inode(struct super_block *sb)
|
|
|
|
{
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
|
|
|
|
if (sbinfo->max_inodes) {
|
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
sbinfo->free_inodes++;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-20 01:00:41 +01:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* shmem_recalc_inode - recalculate the size of an inode
|
|
|
|
* @inode: inode to recalc
|
|
|
|
*
|
|
|
|
* We have to calculate the free blocks since the mm can drop
|
|
|
|
* undirtied hole pages behind our back.
|
|
|
|
*
|
|
|
|
* But normally info->alloced == inode->i_mapping->nrpages + info->swapped
|
|
|
|
* So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
|
|
|
|
*
|
|
|
|
* It has to be called with the spinlock held.
|
|
|
|
*/
|
|
|
|
static void shmem_recalc_inode(struct inode *inode)
|
|
|
|
{
|
|
|
|
struct shmem_inode_info *info = SHMEM_I(inode);
|
|
|
|
long freed;
|
|
|
|
|
|
|
|
freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
|
|
|
|
if (freed > 0) {
|
|
|
|
info->alloced -= freed;
|
|
|
|
shmem_unacct_blocks(info->flags, freed);
|
|
|
|
shmem_free_blocks(inode, freed);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-20 01:00:41 +01:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* shmem_swp_entry - find the swap vector position in the info structure
|
|
|
|
* @info: info structure for the inode
|
|
|
|
* @index: index of the page to find
|
|
|
|
* @page: optional page to add to the structure. Has to be preset to
|
|
|
|
* all zeros
|
|
|
|
*
|
|
|
|
* If there is no space allocated yet it will return NULL when
|
|
|
|
* page is NULL, else it will use the page for the needed block,
|
|
|
|
* setting it to NULL on return to indicate that it has been used.
|
|
|
|
*
|
|
|
|
* The swap vector is organized the following way:
|
|
|
|
*
|
|
|
|
* There are SHMEM_NR_DIRECT entries directly stored in the
|
|
|
|
* shmem_inode_info structure. So small files do not need an addional
|
|
|
|
* allocation.
|
|
|
|
*
|
|
|
|
* For pages with index > SHMEM_NR_DIRECT there is the pointer
|
|
|
|
* i_indirect which points to a page which holds in the first half
|
|
|
|
* doubly indirect blocks, in the second half triple indirect blocks:
|
|
|
|
*
|
|
|
|
* For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
|
|
|
|
* following layout (for SHMEM_NR_DIRECT == 16):
|
|
|
|
*
|
|
|
|
* i_indirect -> dir --> 16-19
|
|
|
|
* | +-> 20-23
|
|
|
|
* |
|
|
|
|
* +-->dir2 --> 24-27
|
|
|
|
* | +-> 28-31
|
|
|
|
* | +-> 32-35
|
|
|
|
* | +-> 36-39
|
|
|
|
* |
|
|
|
|
* +-->dir3 --> 40-43
|
|
|
|
* +-> 44-47
|
|
|
|
* +-> 48-51
|
|
|
|
* +-> 52-55
|
|
|
|
*/
|
|
|
|
static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
|
|
|
|
{
|
|
|
|
unsigned long offset;
|
|
|
|
struct page **dir;
|
|
|
|
struct page *subdir;
|
|
|
|
|
|
|
|
if (index < SHMEM_NR_DIRECT) {
|
|
|
|
shmem_swp_balance_unmap();
|
|
|
|
return info->i_direct+index;
|
|
|
|
}
|
|
|
|
if (!info->i_indirect) {
|
|
|
|
if (page) {
|
|
|
|
info->i_indirect = *page;
|
|
|
|
*page = NULL;
|
|
|
|
}
|
|
|
|
return NULL; /* need another page */
|
|
|
|
}
|
|
|
|
|
|
|
|
index -= SHMEM_NR_DIRECT;
|
|
|
|
offset = index % ENTRIES_PER_PAGE;
|
|
|
|
index /= ENTRIES_PER_PAGE;
|
|
|
|
dir = shmem_dir_map(info->i_indirect);
|
|
|
|
|
|
|
|
if (index >= ENTRIES_PER_PAGE/2) {
|
|
|
|
index -= ENTRIES_PER_PAGE/2;
|
|
|
|
dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
|
|
|
|
index %= ENTRIES_PER_PAGE;
|
|
|
|
subdir = *dir;
|
|
|
|
if (!subdir) {
|
|
|
|
if (page) {
|
|
|
|
*dir = *page;
|
|
|
|
*page = NULL;
|
|
|
|
}
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
return NULL; /* need another page */
|
|
|
|
}
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
dir = shmem_dir_map(subdir);
|
|
|
|
}
|
|
|
|
|
|
|
|
dir += index;
|
|
|
|
subdir = *dir;
|
|
|
|
if (!subdir) {
|
|
|
|
if (!page || !(subdir = *page)) {
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
return NULL; /* need a page */
|
|
|
|
}
|
|
|
|
*dir = subdir;
|
|
|
|
*page = NULL;
|
|
|
|
}
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
return shmem_swp_map(subdir) + offset;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
|
|
|
|
{
|
|
|
|
long incdec = value? 1: -1;
|
|
|
|
|
|
|
|
entry->val = value;
|
|
|
|
info->swapped += incdec;
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 02:16:40 +01:00
|
|
|
if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
|
|
|
|
struct page *page = kmap_atomic_to_page(entry);
|
|
|
|
set_page_private(page, page_private(page) + incdec);
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2008-03-20 01:00:41 +01:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* shmem_swp_alloc - get the position of the swap entry for the page.
|
|
|
|
* @info: info structure for the inode
|
|
|
|
* @index: index of the page to find
|
|
|
|
* @sgp: check and recheck i_size? skip allocation?
|
2008-03-20 01:00:41 +01:00
|
|
|
*
|
|
|
|
* If the entry does not exist, allocate it.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
|
|
|
|
{
|
|
|
|
struct inode *inode = &info->vfs_inode;
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
|
|
|
|
struct page *page = NULL;
|
|
|
|
swp_entry_t *entry;
|
|
|
|
|
|
|
|
if (sgp != SGP_WRITE &&
|
|
|
|
((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
while (!(entry = shmem_swp_entry(info, index, &page))) {
|
|
|
|
if (sgp == SGP_READ)
|
|
|
|
return shmem_swp_map(ZERO_PAGE(0));
|
|
|
|
/*
|
|
|
|
* Test free_blocks against 1 not 0, since we have 1 data
|
|
|
|
* page (and perhaps indirect index pages) yet to allocate:
|
|
|
|
* a waste to allocate index if we cannot allocate data.
|
|
|
|
*/
|
2005-06-22 02:15:04 +02:00
|
|
|
if (sbinfo->max_blocks) {
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (sbinfo->free_blocks <= 1) {
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
return ERR_PTR(-ENOSPC);
|
|
|
|
}
|
|
|
|
sbinfo->free_blocks--;
|
|
|
|
inode->i_blocks += BLOCKS_PER_PAGE;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock(&info->lock);
|
2007-07-17 13:03:05 +02:00
|
|
|
page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 02:16:40 +01:00
|
|
|
if (page)
|
|
|
|
set_page_private(page, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&info->lock);
|
|
|
|
|
|
|
|
if (!page) {
|
|
|
|
shmem_free_blocks(inode, 1);
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
}
|
|
|
|
if (sgp != SGP_WRITE &&
|
|
|
|
((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
|
|
|
|
entry = ERR_PTR(-EINVAL);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (info->next_index <= index)
|
|
|
|
info->next_index = index + 1;
|
|
|
|
}
|
|
|
|
if (page) {
|
|
|
|
/* another task gave its page, or truncated the file */
|
|
|
|
shmem_free_blocks(inode, 1);
|
|
|
|
shmem_dir_free(page);
|
|
|
|
}
|
|
|
|
if (info->next_index <= index && !IS_ERR(entry))
|
|
|
|
info->next_index = index + 1;
|
|
|
|
return entry;
|
|
|
|
}
|
|
|
|
|
2008-03-20 01:00:41 +01:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* shmem_free_swp - free some swap entries in a directory
|
2007-03-29 10:20:36 +02:00
|
|
|
* @dir: pointer to the directory
|
|
|
|
* @edir: pointer after last entry of the directory
|
|
|
|
* @punch_lock: pointer to spinlock when needed for the holepunch case
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2007-03-29 10:20:36 +02:00
|
|
|
static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
|
|
|
|
spinlock_t *punch_lock)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-03-29 10:20:36 +02:00
|
|
|
spinlock_t *punch_unlock = NULL;
|
2005-04-17 00:20:36 +02:00
|
|
|
swp_entry_t *ptr;
|
|
|
|
int freed = 0;
|
|
|
|
|
|
|
|
for (ptr = dir; ptr < edir; ptr++) {
|
|
|
|
if (ptr->val) {
|
2007-03-29 10:20:36 +02:00
|
|
|
if (unlikely(punch_lock)) {
|
|
|
|
punch_unlock = punch_lock;
|
|
|
|
punch_lock = NULL;
|
|
|
|
spin_lock(punch_unlock);
|
|
|
|
if (!ptr->val)
|
|
|
|
continue;
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
free_swap_and_cache(*ptr);
|
|
|
|
*ptr = (swp_entry_t){0};
|
|
|
|
freed++;
|
|
|
|
}
|
|
|
|
}
|
2007-03-29 10:20:36 +02:00
|
|
|
if (punch_unlock)
|
|
|
|
spin_unlock(punch_unlock);
|
2005-04-17 00:20:36 +02:00
|
|
|
return freed;
|
|
|
|
}
|
|
|
|
|
2007-03-29 10:20:36 +02:00
|
|
|
static int shmem_map_and_free_swp(struct page *subdir, int offset,
|
|
|
|
int limit, struct page ***dir, spinlock_t *punch_lock)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
swp_entry_t *ptr;
|
|
|
|
int freed = 0;
|
|
|
|
|
|
|
|
ptr = shmem_swp_map(subdir);
|
|
|
|
for (; offset < limit; offset += LATENCY_LIMIT) {
|
|
|
|
int size = limit - offset;
|
|
|
|
if (size > LATENCY_LIMIT)
|
|
|
|
size = LATENCY_LIMIT;
|
2007-03-29 10:20:36 +02:00
|
|
|
freed += shmem_free_swp(ptr+offset, ptr+offset+size,
|
|
|
|
punch_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (need_resched()) {
|
|
|
|
shmem_swp_unmap(ptr);
|
|
|
|
if (*dir) {
|
|
|
|
shmem_dir_unmap(*dir);
|
|
|
|
*dir = NULL;
|
|
|
|
}
|
|
|
|
cond_resched();
|
|
|
|
ptr = shmem_swp_map(subdir);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
shmem_swp_unmap(ptr);
|
|
|
|
return freed;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void shmem_free_pages(struct list_head *next)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
int freed = 0;
|
|
|
|
|
|
|
|
do {
|
|
|
|
page = container_of(next, struct page, lru);
|
|
|
|
next = next->next;
|
|
|
|
shmem_dir_free(page);
|
|
|
|
freed++;
|
|
|
|
if (freed >= LATENCY_LIMIT) {
|
|
|
|
cond_resched();
|
|
|
|
freed = 0;
|
|
|
|
}
|
|
|
|
} while (next);
|
|
|
|
}
|
|
|
|
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
struct shmem_inode_info *info = SHMEM_I(inode);
|
|
|
|
unsigned long idx;
|
|
|
|
unsigned long size;
|
|
|
|
unsigned long limit;
|
|
|
|
unsigned long stage;
|
|
|
|
unsigned long diroff;
|
|
|
|
struct page **dir;
|
|
|
|
struct page *topdir;
|
|
|
|
struct page *middir;
|
|
|
|
struct page *subdir;
|
|
|
|
swp_entry_t *ptr;
|
|
|
|
LIST_HEAD(pages_to_free);
|
|
|
|
long nr_pages_to_free = 0;
|
|
|
|
long nr_swaps_freed = 0;
|
|
|
|
int offset;
|
|
|
|
int freed;
|
2007-03-29 10:20:35 +02:00
|
|
|
int punch_hole;
|
2007-03-29 10:20:36 +02:00
|
|
|
spinlock_t *needs_lock;
|
|
|
|
spinlock_t *punch_lock;
|
2007-03-29 10:20:35 +02:00
|
|
|
unsigned long upper_limit;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
inode->i_ctime = inode->i_mtime = CURRENT_TIME;
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
|
2005-04-17 00:20:36 +02:00
|
|
|
if (idx >= info->next_index)
|
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
info->flags |= SHMEM_TRUNCATE;
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
if (likely(end == (loff_t) -1)) {
|
|
|
|
limit = info->next_index;
|
2007-03-29 10:20:35 +02:00
|
|
|
upper_limit = SHMEM_MAX_INDEX;
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
info->next_index = idx;
|
2007-03-29 10:20:36 +02:00
|
|
|
needs_lock = NULL;
|
2007-03-29 10:20:35 +02:00
|
|
|
punch_hole = 0;
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
} else {
|
2007-03-29 10:20:35 +02:00
|
|
|
if (end + 1 >= inode->i_size) { /* we may free a little more */
|
|
|
|
limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
|
|
|
|
PAGE_CACHE_SHIFT;
|
|
|
|
upper_limit = SHMEM_MAX_INDEX;
|
|
|
|
} else {
|
|
|
|
limit = (end + 1) >> PAGE_CACHE_SHIFT;
|
|
|
|
upper_limit = limit;
|
|
|
|
}
|
2007-03-29 10:20:36 +02:00
|
|
|
needs_lock = &info->lock;
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
punch_hole = 1;
|
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
topdir = info->i_indirect;
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
|
2005-04-17 00:20:36 +02:00
|
|
|
info->i_indirect = NULL;
|
|
|
|
nr_pages_to_free++;
|
|
|
|
list_add(&topdir->lru, &pages_to_free);
|
|
|
|
}
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
|
|
|
|
if (info->swapped && idx < SHMEM_NR_DIRECT) {
|
|
|
|
ptr = info->i_direct;
|
|
|
|
size = limit;
|
|
|
|
if (size > SHMEM_NR_DIRECT)
|
|
|
|
size = SHMEM_NR_DIRECT;
|
2007-03-29 10:20:36 +02:00
|
|
|
nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-12-22 10:06:23 +01:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If there are no indirect blocks or we are punching a hole
|
|
|
|
* below indirect blocks, nothing to be done.
|
|
|
|
*/
|
2007-03-29 10:20:35 +02:00
|
|
|
if (!topdir || limit <= SHMEM_NR_DIRECT)
|
2005-04-17 00:20:36 +02:00
|
|
|
goto done2;
|
|
|
|
|
2007-03-29 10:20:36 +02:00
|
|
|
/*
|
|
|
|
* The truncation case has already dropped info->lock, and we're safe
|
|
|
|
* because i_size and next_index have already been lowered, preventing
|
|
|
|
* access beyond. But in the punch_hole case, we still need to take
|
|
|
|
* the lock when updating the swap directory, because there might be
|
|
|
|
* racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
|
|
|
|
* shmem_writepage. However, whenever we find we can remove a whole
|
|
|
|
* directory page (not at the misaligned start or end of the range),
|
|
|
|
* we first NULLify its pointer in the level above, and then have no
|
|
|
|
* need to take the lock when updating its contents: needs_lock and
|
|
|
|
* punch_lock (either pointing to info->lock or NULL) manage this.
|
|
|
|
*/
|
|
|
|
|
2007-03-29 10:20:35 +02:00
|
|
|
upper_limit -= SHMEM_NR_DIRECT;
|
2005-04-17 00:20:36 +02:00
|
|
|
limit -= SHMEM_NR_DIRECT;
|
|
|
|
idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
|
|
|
|
offset = idx % ENTRIES_PER_PAGE;
|
|
|
|
idx -= offset;
|
|
|
|
|
|
|
|
dir = shmem_dir_map(topdir);
|
|
|
|
stage = ENTRIES_PER_PAGEPAGE/2;
|
|
|
|
if (idx < ENTRIES_PER_PAGEPAGE/2) {
|
|
|
|
middir = topdir;
|
|
|
|
diroff = idx/ENTRIES_PER_PAGE;
|
|
|
|
} else {
|
|
|
|
dir += ENTRIES_PER_PAGE/2;
|
|
|
|
dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
|
|
|
|
while (stage <= idx)
|
|
|
|
stage += ENTRIES_PER_PAGEPAGE;
|
|
|
|
middir = *dir;
|
|
|
|
if (*dir) {
|
|
|
|
diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
|
|
|
|
ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
|
2007-03-29 10:20:35 +02:00
|
|
|
if (!diroff && !offset && upper_limit >= stage) {
|
2007-03-29 10:20:36 +02:00
|
|
|
if (needs_lock) {
|
|
|
|
spin_lock(needs_lock);
|
|
|
|
*dir = NULL;
|
|
|
|
spin_unlock(needs_lock);
|
|
|
|
needs_lock = NULL;
|
|
|
|
} else
|
|
|
|
*dir = NULL;
|
2005-04-17 00:20:36 +02:00
|
|
|
nr_pages_to_free++;
|
|
|
|
list_add(&middir->lru, &pages_to_free);
|
|
|
|
}
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
dir = shmem_dir_map(middir);
|
|
|
|
} else {
|
|
|
|
diroff = 0;
|
|
|
|
offset = 0;
|
|
|
|
idx = stage;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
|
|
|
|
if (unlikely(idx == stage)) {
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
dir = shmem_dir_map(topdir) +
|
|
|
|
ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
|
|
|
|
while (!*dir) {
|
|
|
|
dir++;
|
|
|
|
idx += ENTRIES_PER_PAGEPAGE;
|
|
|
|
if (idx >= limit)
|
|
|
|
goto done1;
|
|
|
|
}
|
|
|
|
stage = idx + ENTRIES_PER_PAGEPAGE;
|
|
|
|
middir = *dir;
|
2007-03-29 10:20:36 +02:00
|
|
|
if (punch_hole)
|
|
|
|
needs_lock = &info->lock;
|
2007-03-29 10:20:35 +02:00
|
|
|
if (upper_limit >= stage) {
|
2007-03-29 10:20:36 +02:00
|
|
|
if (needs_lock) {
|
|
|
|
spin_lock(needs_lock);
|
|
|
|
*dir = NULL;
|
|
|
|
spin_unlock(needs_lock);
|
|
|
|
needs_lock = NULL;
|
|
|
|
} else
|
|
|
|
*dir = NULL;
|
2007-03-29 10:20:35 +02:00
|
|
|
nr_pages_to_free++;
|
|
|
|
list_add(&middir->lru, &pages_to_free);
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
cond_resched();
|
|
|
|
dir = shmem_dir_map(middir);
|
|
|
|
diroff = 0;
|
|
|
|
}
|
2007-03-29 10:20:36 +02:00
|
|
|
punch_lock = needs_lock;
|
2005-04-17 00:20:36 +02:00
|
|
|
subdir = dir[diroff];
|
2007-03-29 10:20:36 +02:00
|
|
|
if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
|
|
|
|
if (needs_lock) {
|
|
|
|
spin_lock(needs_lock);
|
|
|
|
dir[diroff] = NULL;
|
|
|
|
spin_unlock(needs_lock);
|
|
|
|
punch_lock = NULL;
|
|
|
|
} else
|
|
|
|
dir[diroff] = NULL;
|
|
|
|
nr_pages_to_free++;
|
|
|
|
list_add(&subdir->lru, &pages_to_free);
|
|
|
|
}
|
|
|
|
if (subdir && page_private(subdir) /* has swap entries */) {
|
2005-04-17 00:20:36 +02:00
|
|
|
size = limit - idx;
|
|
|
|
if (size > ENTRIES_PER_PAGE)
|
|
|
|
size = ENTRIES_PER_PAGE;
|
|
|
|
freed = shmem_map_and_free_swp(subdir,
|
2007-03-29 10:20:36 +02:00
|
|
|
offset, size, &dir, punch_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!dir)
|
|
|
|
dir = shmem_dir_map(middir);
|
|
|
|
nr_swaps_freed += freed;
|
2007-03-29 10:20:36 +02:00
|
|
|
if (offset || punch_lock) {
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&info->lock);
|
2007-03-29 10:20:36 +02:00
|
|
|
set_page_private(subdir,
|
|
|
|
page_private(subdir) - freed);
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
2007-03-29 10:20:36 +02:00
|
|
|
} else
|
|
|
|
BUG_ON(page_private(subdir) != freed);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2007-03-29 10:20:36 +02:00
|
|
|
offset = 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
done1:
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
done2:
|
|
|
|
if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
|
|
|
|
/*
|
|
|
|
* Call truncate_inode_pages again: racing shmem_unuse_inode
|
|
|
|
* may have swizzled a page in from swap since vmtruncate or
|
|
|
|
* generic_delete_inode did it, before we lowered next_index.
|
|
|
|
* Also, though shmem_getpage checks i_size before adding to
|
|
|
|
* cache, no recheck after: so fix the narrow window there too.
|
2007-03-29 10:20:37 +02:00
|
|
|
*
|
|
|
|
* Recalling truncate_inode_pages_range and unmap_mapping_range
|
|
|
|
* every time for punch_hole (which never got a chance to clear
|
|
|
|
* SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
|
|
|
|
* yet hardly ever necessary: try to optimize them out later.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
truncate_inode_pages_range(inode->i_mapping, start, end);
|
2007-03-29 10:20:37 +02:00
|
|
|
if (punch_hole)
|
|
|
|
unmap_mapping_range(inode->i_mapping, start,
|
|
|
|
end - start, 1);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
info->flags &= ~SHMEM_TRUNCATE;
|
|
|
|
info->swapped -= nr_swaps_freed;
|
|
|
|
if (nr_pages_to_free)
|
|
|
|
shmem_free_blocks(inode, nr_pages_to_free);
|
|
|
|
shmem_recalc_inode(inode);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Empty swap vector directory pages to be freed?
|
|
|
|
*/
|
|
|
|
if (!list_empty(&pages_to_free)) {
|
|
|
|
pages_to_free.prev->next = NULL;
|
|
|
|
shmem_free_pages(pages_to_free.next);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
static void shmem_truncate(struct inode *inode)
|
|
|
|
{
|
|
|
|
shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
|
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
|
|
|
|
{
|
|
|
|
struct inode *inode = dentry->d_inode;
|
|
|
|
struct page *page = NULL;
|
|
|
|
int error;
|
|
|
|
|
2006-09-29 11:01:35 +02:00
|
|
|
if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
|
2005-04-17 00:20:36 +02:00
|
|
|
if (attr->ia_size < inode->i_size) {
|
|
|
|
/*
|
|
|
|
* If truncating down to a partial page, then
|
|
|
|
* if that page is already allocated, hold it
|
|
|
|
* in memory until the truncation is over, so
|
|
|
|
* truncate_partial_page cannnot miss it were
|
|
|
|
* it assigned to swap.
|
|
|
|
*/
|
|
|
|
if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
|
|
|
|
(void) shmem_getpage(inode,
|
|
|
|
attr->ia_size>>PAGE_CACHE_SHIFT,
|
|
|
|
&page, SGP_READ, NULL);
|
2008-02-05 07:28:44 +01:00
|
|
|
if (page)
|
|
|
|
unlock_page(page);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Reset SHMEM_PAGEIN flag so that shmem_truncate can
|
|
|
|
* detect if any pages might have been added to cache
|
|
|
|
* after truncate_inode_pages. But we needn't bother
|
|
|
|
* if it's being fully truncated to zero-length: the
|
|
|
|
* nrpages check is efficient enough in that case.
|
|
|
|
*/
|
|
|
|
if (attr->ia_size) {
|
|
|
|
struct shmem_inode_info *info = SHMEM_I(inode);
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
info->flags &= ~SHMEM_PAGEIN;
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
error = inode_change_ok(inode, attr);
|
|
|
|
if (!error)
|
|
|
|
error = inode_setattr(inode, attr);
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
|
|
|
if (!error && (attr->ia_valid & ATTR_MODE))
|
|
|
|
error = generic_acl_chmod(inode, &shmem_acl_ops);
|
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
if (page)
|
|
|
|
page_cache_release(page);
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void shmem_delete_inode(struct inode *inode)
|
|
|
|
{
|
|
|
|
struct shmem_inode_info *info = SHMEM_I(inode);
|
|
|
|
|
|
|
|
if (inode->i_op->truncate == shmem_truncate) {
|
2005-09-09 22:01:31 +02:00
|
|
|
truncate_inode_pages(inode->i_mapping, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
shmem_unacct_size(info->flags, inode->i_size);
|
|
|
|
inode->i_size = 0;
|
|
|
|
shmem_truncate(inode);
|
|
|
|
if (!list_empty(&info->swaplist)) {
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
mutex_lock(&shmem_swaplist_mutex);
|
2005-04-17 00:20:36 +02:00
|
|
|
list_del_init(&info->swaplist);
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
mutex_unlock(&shmem_swaplist_mutex);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
}
|
2005-06-22 02:15:04 +02:00
|
|
|
BUG_ON(inode->i_blocks);
|
2008-02-05 07:28:47 +01:00
|
|
|
shmem_free_inode(inode->i_sb);
|
2005-04-17 00:20:36 +02:00
|
|
|
clear_inode(inode);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
|
|
|
|
{
|
|
|
|
swp_entry_t *ptr;
|
|
|
|
|
|
|
|
for (ptr = dir; ptr < edir; ptr++) {
|
|
|
|
if (ptr->val == entry.val)
|
|
|
|
return ptr - dir;
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
unsigned long idx;
|
|
|
|
unsigned long size;
|
|
|
|
unsigned long limit;
|
|
|
|
unsigned long stage;
|
|
|
|
struct page **dir;
|
|
|
|
struct page *subdir;
|
|
|
|
swp_entry_t *ptr;
|
|
|
|
int offset;
|
2008-02-05 07:28:51 +01:00
|
|
|
int error;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
idx = 0;
|
|
|
|
ptr = info->i_direct;
|
|
|
|
spin_lock(&info->lock);
|
tmpfs: fix shmem_swaplist races
Intensive swapoff testing shows shmem_unuse spinning on an entry in
shmem_swaplist pointing to itself: how does that come about? Days pass...
First guess is this: shmem_delete_inode tests list_empty without taking the
global mutex (so the swapping case doesn't slow down the common case); but
there's an instant in shmem_unuse_inode's list_move_tail when the list entry
may appear empty (a rare case, because it's actually moving the head not the
the list member). So there's a danger of leaving the inode on the swaplist
when it's freed, then reinitialized to point to itself when reused. Fix that
by skipping the list_move_tail when it's a no-op, which happens to plug this.
But this same spinning then surfaces on another machine. Ah, I'd never
suspected it, but shmem_writepage's swaplist manipulation is unsafe: though we
still hold page lock, which would hold off inode deletion if the page were in
pagecache, it doesn't hold off once it's in swapcache (free_swap_and_cache
doesn't wait on locked pages). Hmm: we could put the the inode on swaplist
earlier, but then shmem_unuse_inode could never prune unswapped inodes.
Fix this with an igrab before dropping info->lock, as in shmem_unuse_inode;
though I am a little uneasy about the iput which has to follow - it works, and
I see nothing wrong with it, but it is surprising that shmem inode deletion
may now occur below shmem_writepage. Revisit this fix later?
And while we're looking at these races: the way shmem_unuse tests swapped
without holding info->lock looks unsafe, if we've more than one swap area: a
racing shmem_writepage on another page of the same inode could be putting it
in swapcache, just as we're deciding to remove the inode from swaplist -
there's a danger of going on swap without being listed, so a later swapoff
would hang, being unable to locate the entry. Move that test and removal down
into shmem_unuse_inode, once info->lock is held.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:55 +01:00
|
|
|
if (!info->swapped) {
|
|
|
|
list_del_init(&info->swaplist);
|
|
|
|
goto lost2;
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
limit = info->next_index;
|
|
|
|
size = limit;
|
|
|
|
if (size > SHMEM_NR_DIRECT)
|
|
|
|
size = SHMEM_NR_DIRECT;
|
|
|
|
offset = shmem_find_swp(entry, ptr, ptr+size);
|
2008-02-05 07:28:53 +01:00
|
|
|
if (offset >= 0)
|
2005-04-17 00:20:36 +02:00
|
|
|
goto found;
|
|
|
|
if (!info->i_indirect)
|
|
|
|
goto lost2;
|
|
|
|
|
|
|
|
dir = shmem_dir_map(info->i_indirect);
|
|
|
|
stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
|
|
|
|
|
|
|
|
for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
|
|
|
|
if (unlikely(idx == stage)) {
|
|
|
|
shmem_dir_unmap(dir-1);
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
if (cond_resched_lock(&info->lock)) {
|
|
|
|
/* check it has not been truncated */
|
|
|
|
if (limit > info->next_index) {
|
|
|
|
limit = info->next_index;
|
|
|
|
if (idx >= limit)
|
|
|
|
goto lost2;
|
|
|
|
}
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
dir = shmem_dir_map(info->i_indirect) +
|
|
|
|
ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
|
|
|
|
while (!*dir) {
|
|
|
|
dir++;
|
|
|
|
idx += ENTRIES_PER_PAGEPAGE;
|
|
|
|
if (idx >= limit)
|
|
|
|
goto lost1;
|
|
|
|
}
|
|
|
|
stage = idx + ENTRIES_PER_PAGEPAGE;
|
|
|
|
subdir = *dir;
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
dir = shmem_dir_map(subdir);
|
|
|
|
}
|
|
|
|
subdir = *dir;
|
[PATCH] mm: split page table lock
Christoph Lameter demonstrated very poor scalability on the SGI 512-way, with
a many-threaded application which concurrently initializes different parts of
a large anonymous area.
This patch corrects that, by using a separate spinlock per page table page, to
guard the page table entries in that page, instead of using the mm's single
page_table_lock. (But even then, page_table_lock is still used to guard page
table allocation, and anon_vma allocation.)
In this implementation, the spinlock is tucked inside the struct page of the
page table page: with a BUILD_BUG_ON in case it overflows - which it would in
the case of 32-bit PA-RISC with spinlock debugging enabled.
Splitting the lock is not quite for free: another cacheline access. Ideally,
I suppose we would use split ptlock only for multi-threaded processes on
multi-cpu machines; but deciding that dynamically would have its own costs.
So for now enable it by config, at some number of cpus - since the Kconfig
language doesn't support inequalities, let preprocessor compare that with
NR_CPUS. But I don't think it's worth being user-configurable: for good
testing of both split and unsplit configs, split now at 4 cpus, and perhaps
change that to 8 later.
There is a benefit even for singly threaded processes: kswapd can be attacking
one part of the mm while another part is busy faulting.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 02:16:40 +01:00
|
|
|
if (subdir && page_private(subdir)) {
|
2005-04-17 00:20:36 +02:00
|
|
|
ptr = shmem_swp_map(subdir);
|
|
|
|
size = limit - idx;
|
|
|
|
if (size > ENTRIES_PER_PAGE)
|
|
|
|
size = ENTRIES_PER_PAGE;
|
|
|
|
offset = shmem_find_swp(entry, ptr, ptr+size);
|
2008-02-05 07:28:53 +01:00
|
|
|
shmem_swp_unmap(ptr);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (offset >= 0) {
|
|
|
|
shmem_dir_unmap(dir);
|
|
|
|
goto found;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
lost1:
|
|
|
|
shmem_dir_unmap(dir-1);
|
|
|
|
lost2:
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
return 0;
|
|
|
|
found:
|
|
|
|
idx += offset;
|
2008-02-05 07:28:53 +01:00
|
|
|
inode = igrab(&info->vfs_inode);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
|
tmpfs: fix shmem_swaplist races
Intensive swapoff testing shows shmem_unuse spinning on an entry in
shmem_swaplist pointing to itself: how does that come about? Days pass...
First guess is this: shmem_delete_inode tests list_empty without taking the
global mutex (so the swapping case doesn't slow down the common case); but
there's an instant in shmem_unuse_inode's list_move_tail when the list entry
may appear empty (a rare case, because it's actually moving the head not the
the list member). So there's a danger of leaving the inode on the swaplist
when it's freed, then reinitialized to point to itself when reused. Fix that
by skipping the list_move_tail when it's a no-op, which happens to plug this.
But this same spinning then surfaces on another machine. Ah, I'd never
suspected it, but shmem_writepage's swaplist manipulation is unsafe: though we
still hold page lock, which would hold off inode deletion if the page were in
pagecache, it doesn't hold off once it's in swapcache (free_swap_and_cache
doesn't wait on locked pages). Hmm: we could put the the inode on swaplist
earlier, but then shmem_unuse_inode could never prune unswapped inodes.
Fix this with an igrab before dropping info->lock, as in shmem_unuse_inode;
though I am a little uneasy about the iput which has to follow - it works, and
I see nothing wrong with it, but it is surprising that shmem inode deletion
may now occur below shmem_writepage. Revisit this fix later?
And while we're looking at these races: the way shmem_unuse tests swapped
without holding info->lock looks unsafe, if we've more than one swap area: a
racing shmem_writepage on another page of the same inode could be putting it
in swapcache, just as we're deciding to remove the inode from swaplist -
there's a danger of going on swap without being listed, so a later swapoff
would hang, being unable to locate the entry. Move that test and removal down
into shmem_unuse_inode, once info->lock is held.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:55 +01:00
|
|
|
/*
|
|
|
|
* Move _head_ to start search for next from here.
|
|
|
|
* But be careful: shmem_delete_inode checks list_empty without taking
|
|
|
|
* mutex, and there's an instant in list_move_tail when info->swaplist
|
|
|
|
* would appear empty, if it were the only one on shmem_swaplist. We
|
|
|
|
* could avoid doing it if inode NULL; or use this minor optimization.
|
|
|
|
*/
|
|
|
|
if (shmem_swaplist.next != &info->swaplist)
|
|
|
|
list_move_tail(&shmem_swaplist, &info->swaplist);
|
2008-02-05 07:28:53 +01:00
|
|
|
mutex_unlock(&shmem_swaplist_mutex);
|
|
|
|
|
|
|
|
error = 1;
|
|
|
|
if (!inode)
|
|
|
|
goto out;
|
2009-01-08 03:07:56 +01:00
|
|
|
/*
|
2009-01-08 03:08:35 +01:00
|
|
|
* Charge page using GFP_KERNEL while we can wait.
|
|
|
|
* Charged back to the user(not to caller) when swap account is used.
|
|
|
|
* add_to_page_cache() will be called with GFP_NOWAIT.
|
2009-01-08 03:07:56 +01:00
|
|
|
*/
|
2008-02-07 09:14:22 +01:00
|
|
|
error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
|
2008-02-05 07:28:54 +01:00
|
|
|
if (error)
|
|
|
|
goto out;
|
2008-02-07 09:14:22 +01:00
|
|
|
error = radix_tree_preload(GFP_KERNEL);
|
memcg: remove refcnt from page_cgroup
memcg: performance improvements
Patch Description
1/5 ... remove refcnt fron page_cgroup patch (shmem handling is fixed)
2/5 ... swapcache handling patch
3/5 ... add helper function for shmem's memory reclaim patch
4/5 ... optimize by likely/unlikely ppatch
5/5 ... remove redundunt check patch (shmem handling is fixed.)
Unix bench result.
== 2.6.26-rc2-mm1 + memory resource controller
Execl Throughput 2915.4 lps (29.6 secs, 3 samples)
C Compiler Throughput 1019.3 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5796.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1097.7 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 565.3 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1022128.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 544057.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 346481.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 319325.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 148788.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 99051.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2058917.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1606109.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 854789.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 126145.2 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 2915.4 678.0
File Copy 1024 bufsize 2000 maxblocks 3960.0 346481.0 875.0
File Copy 256 bufsize 500 maxblocks 1655.0 99051.0 598.5
File Copy 4096 bufsize 8000 maxblocks 5800.0 854789.0 1473.8
Shell Scripts (8 concurrent) 6.0 1097.7 1829.5
=========
FINAL SCORE 991.3
== 2.6.26-rc2-mm1 + this set ==
Execl Throughput 3012.9 lps (29.9 secs, 3 samples)
C Compiler Throughput 981.0 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5872.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1120.3 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 578.0 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1003993.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 550452.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 347159.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 314644.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 151852.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 101000.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2033256.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1611814.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 847979.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 128148.7 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 3012.9 700.7
File Copy 1024 bufsize 2000 maxblocks 3960.0 347159.0 876.7
File Copy 256 bufsize 500 maxblocks 1655.0 101000.0 610.3
File Copy 4096 bufsize 8000 maxblocks 5800.0 847979.0 1462.0
Shell Scripts (8 concurrent) 6.0 1120.3 1867.2
=========
FINAL SCORE 1004.6
This patch:
Remove refcnt from page_cgroup().
After this,
* A page is charged only when !page_mapped() && no page_cgroup is assigned.
* Anon page is newly mapped.
* File page is added to mapping->tree.
* A page is uncharged only when
* Anon page is fully unmapped.
* File page is removed from LRU.
There is no change in behavior from user's view.
This patch also removes unnecessary calls in rmap.c which was used only for
refcnt mangement.
[akpm@linux-foundation.org: fix warning]
[hugh@veritas.com: fix shmem_unuse_inode charging]
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: YAMAMOTO Takashi <yamamoto@valinux.co.jp>
Cc: Paul Menage <menage@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:14 +02:00
|
|
|
if (error) {
|
|
|
|
mem_cgroup_uncharge_cache_page(page);
|
|
|
|
goto out;
|
|
|
|
}
|
2008-02-05 07:28:54 +01:00
|
|
|
error = 1;
|
2008-02-05 07:28:53 +01:00
|
|
|
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
ptr = shmem_swp_entry(info, idx, NULL);
|
memcg: remove refcnt from page_cgroup
memcg: performance improvements
Patch Description
1/5 ... remove refcnt fron page_cgroup patch (shmem handling is fixed)
2/5 ... swapcache handling patch
3/5 ... add helper function for shmem's memory reclaim patch
4/5 ... optimize by likely/unlikely ppatch
5/5 ... remove redundunt check patch (shmem handling is fixed.)
Unix bench result.
== 2.6.26-rc2-mm1 + memory resource controller
Execl Throughput 2915.4 lps (29.6 secs, 3 samples)
C Compiler Throughput 1019.3 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5796.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1097.7 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 565.3 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1022128.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 544057.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 346481.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 319325.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 148788.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 99051.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2058917.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1606109.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 854789.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 126145.2 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 2915.4 678.0
File Copy 1024 bufsize 2000 maxblocks 3960.0 346481.0 875.0
File Copy 256 bufsize 500 maxblocks 1655.0 99051.0 598.5
File Copy 4096 bufsize 8000 maxblocks 5800.0 854789.0 1473.8
Shell Scripts (8 concurrent) 6.0 1097.7 1829.5
=========
FINAL SCORE 991.3
== 2.6.26-rc2-mm1 + this set ==
Execl Throughput 3012.9 lps (29.9 secs, 3 samples)
C Compiler Throughput 981.0 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5872.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1120.3 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 578.0 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1003993.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 550452.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 347159.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 314644.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 151852.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 101000.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2033256.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1611814.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 847979.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 128148.7 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 3012.9 700.7
File Copy 1024 bufsize 2000 maxblocks 3960.0 347159.0 876.7
File Copy 256 bufsize 500 maxblocks 1655.0 101000.0 610.3
File Copy 4096 bufsize 8000 maxblocks 5800.0 847979.0 1462.0
Shell Scripts (8 concurrent) 6.0 1120.3 1867.2
=========
FINAL SCORE 1004.6
This patch:
Remove refcnt from page_cgroup().
After this,
* A page is charged only when !page_mapped() && no page_cgroup is assigned.
* Anon page is newly mapped.
* File page is added to mapping->tree.
* A page is uncharged only when
* Anon page is fully unmapped.
* File page is removed from LRU.
There is no change in behavior from user's view.
This patch also removes unnecessary calls in rmap.c which was used only for
refcnt mangement.
[akpm@linux-foundation.org: fix warning]
[hugh@veritas.com: fix shmem_unuse_inode charging]
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: YAMAMOTO Takashi <yamamoto@valinux.co.jp>
Cc: Paul Menage <menage@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:14 +02:00
|
|
|
if (ptr && ptr->val == entry.val) {
|
2008-07-26 04:45:30 +02:00
|
|
|
error = add_to_page_cache_locked(page, inode->i_mapping,
|
2008-02-05 07:28:54 +01:00
|
|
|
idx, GFP_NOWAIT);
|
memcg: remove refcnt from page_cgroup
memcg: performance improvements
Patch Description
1/5 ... remove refcnt fron page_cgroup patch (shmem handling is fixed)
2/5 ... swapcache handling patch
3/5 ... add helper function for shmem's memory reclaim patch
4/5 ... optimize by likely/unlikely ppatch
5/5 ... remove redundunt check patch (shmem handling is fixed.)
Unix bench result.
== 2.6.26-rc2-mm1 + memory resource controller
Execl Throughput 2915.4 lps (29.6 secs, 3 samples)
C Compiler Throughput 1019.3 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5796.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1097.7 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 565.3 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1022128.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 544057.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 346481.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 319325.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 148788.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 99051.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2058917.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1606109.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 854789.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 126145.2 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 2915.4 678.0
File Copy 1024 bufsize 2000 maxblocks 3960.0 346481.0 875.0
File Copy 256 bufsize 500 maxblocks 1655.0 99051.0 598.5
File Copy 4096 bufsize 8000 maxblocks 5800.0 854789.0 1473.8
Shell Scripts (8 concurrent) 6.0 1097.7 1829.5
=========
FINAL SCORE 991.3
== 2.6.26-rc2-mm1 + this set ==
Execl Throughput 3012.9 lps (29.9 secs, 3 samples)
C Compiler Throughput 981.0 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5872.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1120.3 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 578.0 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1003993.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 550452.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 347159.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 314644.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 151852.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 101000.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2033256.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1611814.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 847979.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 128148.7 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 3012.9 700.7
File Copy 1024 bufsize 2000 maxblocks 3960.0 347159.0 876.7
File Copy 256 bufsize 500 maxblocks 1655.0 101000.0 610.3
File Copy 4096 bufsize 8000 maxblocks 5800.0 847979.0 1462.0
Shell Scripts (8 concurrent) 6.0 1120.3 1867.2
=========
FINAL SCORE 1004.6
This patch:
Remove refcnt from page_cgroup().
After this,
* A page is charged only when !page_mapped() && no page_cgroup is assigned.
* Anon page is newly mapped.
* File page is added to mapping->tree.
* A page is uncharged only when
* Anon page is fully unmapped.
* File page is removed from LRU.
There is no change in behavior from user's view.
This patch also removes unnecessary calls in rmap.c which was used only for
refcnt mangement.
[akpm@linux-foundation.org: fix warning]
[hugh@veritas.com: fix shmem_unuse_inode charging]
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: YAMAMOTO Takashi <yamamoto@valinux.co.jp>
Cc: Paul Menage <menage@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:14 +02:00
|
|
|
/* does mem_cgroup_uncharge_cache_page on error */
|
|
|
|
} else /* we must compensate for our precharge above */
|
|
|
|
mem_cgroup_uncharge_cache_page(page);
|
|
|
|
|
2008-02-05 07:28:51 +01:00
|
|
|
if (error == -EEXIST) {
|
|
|
|
struct page *filepage = find_get_page(inode->i_mapping, idx);
|
2008-02-05 07:28:53 +01:00
|
|
|
error = 1;
|
2008-02-05 07:28:51 +01:00
|
|
|
if (filepage) {
|
|
|
|
/*
|
|
|
|
* There might be a more uptodate page coming down
|
|
|
|
* from a stacked writepage: forget our swappage if so.
|
|
|
|
*/
|
|
|
|
if (PageUptodate(filepage))
|
|
|
|
error = 0;
|
|
|
|
page_cache_release(filepage);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!error) {
|
2008-02-05 07:28:50 +01:00
|
|
|
delete_from_swap_cache(page);
|
|
|
|
set_page_dirty(page);
|
2005-04-17 00:20:36 +02:00
|
|
|
info->flags |= SHMEM_PAGEIN;
|
2008-02-05 07:28:53 +01:00
|
|
|
shmem_swp_set(info, ptr, 0);
|
|
|
|
swap_free(entry);
|
|
|
|
error = 1; /* not an error, but entry was found */
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2008-02-05 07:28:53 +01:00
|
|
|
if (ptr)
|
|
|
|
shmem_swp_unmap(ptr);
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
2008-02-05 07:28:54 +01:00
|
|
|
radix_tree_preload_end();
|
2008-02-05 07:28:53 +01:00
|
|
|
out:
|
|
|
|
unlock_page(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
iput(inode); /* allows for NULL */
|
|
|
|
return error;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* shmem_unuse() search for an eventually swapped out shmem page.
|
|
|
|
*/
|
|
|
|
int shmem_unuse(swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
struct list_head *p, *next;
|
|
|
|
struct shmem_inode_info *info;
|
|
|
|
int found = 0;
|
|
|
|
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
mutex_lock(&shmem_swaplist_mutex);
|
2005-04-17 00:20:36 +02:00
|
|
|
list_for_each_safe(p, next, &shmem_swaplist) {
|
|
|
|
info = list_entry(p, struct shmem_inode_info, swaplist);
|
tmpfs: fix shmem_swaplist races
Intensive swapoff testing shows shmem_unuse spinning on an entry in
shmem_swaplist pointing to itself: how does that come about? Days pass...
First guess is this: shmem_delete_inode tests list_empty without taking the
global mutex (so the swapping case doesn't slow down the common case); but
there's an instant in shmem_unuse_inode's list_move_tail when the list entry
may appear empty (a rare case, because it's actually moving the head not the
the list member). So there's a danger of leaving the inode on the swaplist
when it's freed, then reinitialized to point to itself when reused. Fix that
by skipping the list_move_tail when it's a no-op, which happens to plug this.
But this same spinning then surfaces on another machine. Ah, I'd never
suspected it, but shmem_writepage's swaplist manipulation is unsafe: though we
still hold page lock, which would hold off inode deletion if the page were in
pagecache, it doesn't hold off once it's in swapcache (free_swap_and_cache
doesn't wait on locked pages). Hmm: we could put the the inode on swaplist
earlier, but then shmem_unuse_inode could never prune unswapped inodes.
Fix this with an igrab before dropping info->lock, as in shmem_unuse_inode;
though I am a little uneasy about the iput which has to follow - it works, and
I see nothing wrong with it, but it is surprising that shmem inode deletion
may now occur below shmem_writepage. Revisit this fix later?
And while we're looking at these races: the way shmem_unuse tests swapped
without holding info->lock looks unsafe, if we've more than one swap area: a
racing shmem_writepage on another page of the same inode could be putting it
in swapcache, just as we're deciding to remove the inode from swaplist -
there's a danger of going on swap without being listed, so a later swapoff
would hang, being unable to locate the entry. Move that test and removal down
into shmem_unuse_inode, once info->lock is held.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:55 +01:00
|
|
|
found = shmem_unuse_inode(info, entry, page);
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
cond_resched();
|
2008-02-05 07:28:53 +01:00
|
|
|
if (found)
|
|
|
|
goto out;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
tmpfs: make shmem_unuse more preemptible
shmem_unuse is at present an unbroken search through every swap vector page of
every tmpfs file which might be swapped, all under shmem_swaplist_lock. This
dates from long ago, when the caller held mmlist_lock over it all too: long
gone, but there's never been much pressure for preemptible swapoff.
Make it a little more preemptible, replacing shmem_swaplist_lock by
shmem_swaplist_mutex, inserting a cond_resched in the main loop, and a
cond_resched_lock (on info->lock) at one convenient point in the
shmem_unuse_inode loop, where it has no outstanding kmap_atomic.
If we're serious about preemptible swapoff, there's much further to go e.g.
I'm stupid to let the kmap_atomics of the decreasingly significant HIGHMEM
case dictate preemptiblility for other configs. But as in the earlier patch
to make swapoff scan ptes preemptibly, my hidden agenda is really towards
making memcgroups work, hardly about preemptibility at all.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:52 +01:00
|
|
|
mutex_unlock(&shmem_swaplist_mutex);
|
2008-02-05 07:28:53 +01:00
|
|
|
out: return found; /* 0 or 1 or -ENOMEM */
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Move the page from the page cache to the swap cache.
|
|
|
|
*/
|
|
|
|
static int shmem_writepage(struct page *page, struct writeback_control *wbc)
|
|
|
|
{
|
|
|
|
struct shmem_inode_info *info;
|
|
|
|
swp_entry_t *entry, swap;
|
|
|
|
struct address_space *mapping;
|
|
|
|
unsigned long index;
|
|
|
|
struct inode *inode;
|
|
|
|
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
|
|
mapping = page->mapping;
|
|
|
|
index = page->index;
|
|
|
|
inode = mapping->host;
|
|
|
|
info = SHMEM_I(inode);
|
|
|
|
if (info->flags & VM_LOCKED)
|
|
|
|
goto redirty;
|
2008-02-05 07:28:51 +01:00
|
|
|
if (!total_swap_pages)
|
2005-04-17 00:20:36 +02:00
|
|
|
goto redirty;
|
|
|
|
|
2008-02-05 07:28:51 +01:00
|
|
|
/*
|
|
|
|
* shmem_backing_dev_info's capabilities prevent regular writeback or
|
|
|
|
* sync from ever calling shmem_writepage; but a stacking filesystem
|
|
|
|
* may use the ->writepage of its underlying filesystem, in which case
|
|
|
|
* tmpfs should write out to swap only in response to memory pressure,
|
|
|
|
* and not for pdflush or sync. However, in those cases, we do still
|
|
|
|
* want to check if there's a redundant swappage to be discarded.
|
|
|
|
*/
|
|
|
|
if (wbc->for_reclaim)
|
|
|
|
swap = get_swap_page();
|
|
|
|
else
|
|
|
|
swap.val = 0;
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&info->lock);
|
|
|
|
if (index >= info->next_index) {
|
|
|
|
BUG_ON(!(info->flags & SHMEM_TRUNCATE));
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
entry = shmem_swp_entry(info, index, NULL);
|
2008-02-05 07:28:51 +01:00
|
|
|
if (entry->val) {
|
|
|
|
/*
|
|
|
|
* The more uptodate page coming down from a stacked
|
|
|
|
* writepage should replace our old swappage.
|
|
|
|
*/
|
|
|
|
free_swap_and_cache(*entry);
|
|
|
|
shmem_swp_set(info, entry, 0);
|
|
|
|
}
|
|
|
|
shmem_recalc_inode(inode);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-02-05 07:28:51 +01:00
|
|
|
if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
|
2008-02-05 07:28:50 +01:00
|
|
|
remove_from_page_cache(page);
|
2005-04-17 00:20:36 +02:00
|
|
|
shmem_swp_set(info, entry, swap.val);
|
|
|
|
shmem_swp_unmap(entry);
|
tmpfs: fix shmem_swaplist races
Intensive swapoff testing shows shmem_unuse spinning on an entry in
shmem_swaplist pointing to itself: how does that come about? Days pass...
First guess is this: shmem_delete_inode tests list_empty without taking the
global mutex (so the swapping case doesn't slow down the common case); but
there's an instant in shmem_unuse_inode's list_move_tail when the list entry
may appear empty (a rare case, because it's actually moving the head not the
the list member). So there's a danger of leaving the inode on the swaplist
when it's freed, then reinitialized to point to itself when reused. Fix that
by skipping the list_move_tail when it's a no-op, which happens to plug this.
But this same spinning then surfaces on another machine. Ah, I'd never
suspected it, but shmem_writepage's swaplist manipulation is unsafe: though we
still hold page lock, which would hold off inode deletion if the page were in
pagecache, it doesn't hold off once it's in swapcache (free_swap_and_cache
doesn't wait on locked pages). Hmm: we could put the the inode on swaplist
earlier, but then shmem_unuse_inode could never prune unswapped inodes.
Fix this with an igrab before dropping info->lock, as in shmem_unuse_inode;
though I am a little uneasy about the iput which has to follow - it works, and
I see nothing wrong with it, but it is surprising that shmem inode deletion
may now occur below shmem_writepage. Revisit this fix later?
And while we're looking at these races: the way shmem_unuse tests swapped
without holding info->lock looks unsafe, if we've more than one swap area: a
racing shmem_writepage on another page of the same inode could be putting it
in swapcache, just as we're deciding to remove the inode from swaplist -
there's a danger of going on swap without being listed, so a later swapoff
would hang, being unable to locate the entry. Move that test and removal down
into shmem_unuse_inode, once info->lock is held.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:55 +01:00
|
|
|
if (list_empty(&info->swaplist))
|
|
|
|
inode = igrab(inode);
|
|
|
|
else
|
|
|
|
inode = NULL;
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
2008-02-05 07:28:50 +01:00
|
|
|
swap_duplicate(swap);
|
2008-02-05 07:28:51 +01:00
|
|
|
BUG_ON(page_mapped(page));
|
2008-02-05 07:28:50 +01:00
|
|
|
page_cache_release(page); /* pagecache ref */
|
shmem: writepage directly to swap
Synopsis: if shmem_writepage calls swap_writepage directly, most shmem
swap loads benefit, and a catastrophic interaction between SLUB and some
flash storage is avoided.
shmem_writepage() has always been peculiar in making no attempt to write:
it has just transferred a shmem page from file cache to swap cache, then
let that page make its way around the LRU again before being written and
freed.
The idea was that people use tmpfs because they want those pages to stay
in RAM; so although we give it an overflow to swap, we should resist
writing too soon, giving those pages a second chance before they can be
reclaimed.
That was always questionable, and I've toyed with this patch for years;
but never had a clear justification to depart from the original design.
It became more questionable in 2.6.28, when the split LRU patches classed
shmem and tmpfs pages as SwapBacked rather than as file_cache: that in
itself gives them more resistance to reclaim than normal file pages. I
prepared this patch for 2.6.29, but the merge window arrived before I'd
completed gathering statistics to justify sending it in.
Then while comparing SLQB against SLUB, running SLUB on a laptop I'd
habitually used with SLAB, I found SLUB to run my tmpfs kbuild swapping
tests five times slower than SLAB or SLQB - other machines slower too, but
nowhere near so bad. Simpler "cp -a" swapping tests showed the same.
slub_max_order=0 brings sanity to all, but heavy swapping is too far from
normal to justify such a tuning. The crucial factor on that laptop turns
out to be that I'm using an SD card for swap. What happens is this:
By default, SLUB uses order-2 pages for shmem_inode_cache (and many other
fs inodes), so creating tmpfs files under memory pressure brings lumpy
reclaim into play. One subpage of the order is chosen from the bottom of
the LRU as usual, then the other three picked out from their random
positions on the LRUs.
In a tmpfs load, many of these pages will be ones which already passed
through shmem_writepage, so already have swap allocated. And though their
offsets on swap were probably allocated sequentially, now that the pages
are picked off at random, their swap offsets are scattered.
But the flash storage on the SD card is very sensitive to having its
writes merged: once swap is written at scattered offsets, performance
falls apart. Rotating disk seeks increase too, but less disastrously.
So: stop giving shmem/tmpfs pages a second pass around the LRU, write them
out to swap as soon as their swap has been allocated.
It's surely possible to devise an artificial load which runs faster the
old way, one whose sizing is such that the tmpfs pages on their second
pass are the ones that are wanted again, and other pages not.
But I've not yet found such a load: on all machines, under the loads I've
tried, immediate swap_writepage speeds up shmem swapping: especially when
using the SLUB allocator (and more effectively than slub_max_order=0), but
also with the others; and it also reduces the variance between runs. How
much faster varies widely: a factor of five is rare, 5% is common.
One load which might have suffered: imagine a swapping shmem load in a
limited mem_cgroup on a machine with plenty of memory. Before 2.6.29 the
swapcache was not charged, and such a load would have run quickest with
the shmem swapcache never written to swap. But now swapcache is charged,
so even this load benefits from shmem_writepage directly to swap.
Apologies for the #ifndef CONFIG_SWAP swap_writepage() stub in swap.h:
it's silly because that will never get called; but refactoring shmem.c
sensibly according to CONFIG_SWAP will be a separate task.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-01 00:23:33 +02:00
|
|
|
swap_writepage(page, wbc);
|
tmpfs: fix shmem_swaplist races
Intensive swapoff testing shows shmem_unuse spinning on an entry in
shmem_swaplist pointing to itself: how does that come about? Days pass...
First guess is this: shmem_delete_inode tests list_empty without taking the
global mutex (so the swapping case doesn't slow down the common case); but
there's an instant in shmem_unuse_inode's list_move_tail when the list entry
may appear empty (a rare case, because it's actually moving the head not the
the list member). So there's a danger of leaving the inode on the swaplist
when it's freed, then reinitialized to point to itself when reused. Fix that
by skipping the list_move_tail when it's a no-op, which happens to plug this.
But this same spinning then surfaces on another machine. Ah, I'd never
suspected it, but shmem_writepage's swaplist manipulation is unsafe: though we
still hold page lock, which would hold off inode deletion if the page were in
pagecache, it doesn't hold off once it's in swapcache (free_swap_and_cache
doesn't wait on locked pages). Hmm: we could put the the inode on swaplist
earlier, but then shmem_unuse_inode could never prune unswapped inodes.
Fix this with an igrab before dropping info->lock, as in shmem_unuse_inode;
though I am a little uneasy about the iput which has to follow - it works, and
I see nothing wrong with it, but it is surprising that shmem inode deletion
may now occur below shmem_writepage. Revisit this fix later?
And while we're looking at these races: the way shmem_unuse tests swapped
without holding info->lock looks unsafe, if we've more than one swap area: a
racing shmem_writepage on another page of the same inode could be putting it
in swapcache, just as we're deciding to remove the inode from swaplist -
there's a danger of going on swap without being listed, so a later swapoff
would hang, being unable to locate the entry. Move that test and removal down
into shmem_unuse_inode, once info->lock is held.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:55 +01:00
|
|
|
if (inode) {
|
|
|
|
mutex_lock(&shmem_swaplist_mutex);
|
|
|
|
/* move instead of add in case we're racing */
|
|
|
|
list_move_tail(&info->swaplist, &shmem_swaplist);
|
|
|
|
mutex_unlock(&shmem_swaplist_mutex);
|
|
|
|
iput(inode);
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
unlock:
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
swap_free(swap);
|
|
|
|
redirty:
|
|
|
|
set_page_dirty(page);
|
2008-02-05 07:28:51 +01:00
|
|
|
if (wbc->for_reclaim)
|
|
|
|
return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
|
|
|
|
unlock_page(page);
|
|
|
|
return 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA
|
2008-02-08 13:21:48 +01:00
|
|
|
#ifdef CONFIG_TMPFS
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
|
2008-02-08 13:21:48 +01:00
|
|
|
{
|
mempolicy: rework shmem mpol parsing and display
mm/shmem.c currently contains functions to parse and display memory policy
strings for the tmpfs 'mpol' mount option. Move this to mm/mempolicy.c with
the rest of the mempolicy support. With subsequent patches, we'll be able to
remove knowledge of the details [mode, flags, policy, ...] completely from
shmem.c
1) replace shmem_parse_mpol() in mm/shmem.c with mpol_parse_str() in
mm/mempolicy.c. Rework to use the policy_types[] array [used by
mpol_to_str()] to look up mode by name.
2) use mpol_to_str() to format policy for shmem_show_mpol(). mpol_to_str()
expects a pointer to a struct mempolicy, so temporarily construct one.
This will be replaced with a reference to a struct mempolicy in the tmpfs
superblock in a subsequent patch.
NOTE 1: I changed mpol_to_str() to use a colon ':' rather than an equal
sign '=' as the nodemask delimiter to match mpol_parse_str() and the
tmpfs/shmem mpol mount option formatting that now uses mpol_to_str(). This
is a user visible change to numa_maps, but then the addition of the mode
flags already changed the display. It makes sense to me to have the mounts
and numa_maps display the policy in the same format. However, if anyone
objects strongly, I can pass the desired nodemask delimeter as an arg to
mpol_to_str().
Note 2: Like show_numa_map(), I don't check the return code from
mpol_to_str(). I do use a longer buffer than the one provided by
show_numa_map(), which seems to have sufficed so far.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:23 +02:00
|
|
|
char buffer[64];
|
2008-02-08 13:21:48 +01:00
|
|
|
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
if (!mpol || mpol->mode == MPOL_DEFAULT)
|
mempolicy: rework shmem mpol parsing and display
mm/shmem.c currently contains functions to parse and display memory policy
strings for the tmpfs 'mpol' mount option. Move this to mm/mempolicy.c with
the rest of the mempolicy support. With subsequent patches, we'll be able to
remove knowledge of the details [mode, flags, policy, ...] completely from
shmem.c
1) replace shmem_parse_mpol() in mm/shmem.c with mpol_parse_str() in
mm/mempolicy.c. Rework to use the policy_types[] array [used by
mpol_to_str()] to look up mode by name.
2) use mpol_to_str() to format policy for shmem_show_mpol(). mpol_to_str()
expects a pointer to a struct mempolicy, so temporarily construct one.
This will be replaced with a reference to a struct mempolicy in the tmpfs
superblock in a subsequent patch.
NOTE 1: I changed mpol_to_str() to use a colon ':' rather than an equal
sign '=' as the nodemask delimiter to match mpol_parse_str() and the
tmpfs/shmem mpol mount option formatting that now uses mpol_to_str(). This
is a user visible change to numa_maps, but then the addition of the mode
flags already changed the display. It makes sense to me to have the mounts
and numa_maps display the policy in the same format. However, if anyone
objects strongly, I can pass the desired nodemask delimeter as an arg to
mpol_to_str().
Note 2: Like show_numa_map(), I don't check the return code from
mpol_to_str(). I do use a longer buffer than the one provided by
show_numa_map(), which seems to have sufficed so far.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:23 +02:00
|
|
|
return; /* show nothing */
|
2008-02-08 13:21:48 +01:00
|
|
|
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
mpol_to_str(buffer, sizeof(buffer), mpol, 1);
|
mempolicy: rework shmem mpol parsing and display
mm/shmem.c currently contains functions to parse and display memory policy
strings for the tmpfs 'mpol' mount option. Move this to mm/mempolicy.c with
the rest of the mempolicy support. With subsequent patches, we'll be able to
remove knowledge of the details [mode, flags, policy, ...] completely from
shmem.c
1) replace shmem_parse_mpol() in mm/shmem.c with mpol_parse_str() in
mm/mempolicy.c. Rework to use the policy_types[] array [used by
mpol_to_str()] to look up mode by name.
2) use mpol_to_str() to format policy for shmem_show_mpol(). mpol_to_str()
expects a pointer to a struct mempolicy, so temporarily construct one.
This will be replaced with a reference to a struct mempolicy in the tmpfs
superblock in a subsequent patch.
NOTE 1: I changed mpol_to_str() to use a colon ':' rather than an equal
sign '=' as the nodemask delimiter to match mpol_parse_str() and the
tmpfs/shmem mpol mount option formatting that now uses mpol_to_str(). This
is a user visible change to numa_maps, but then the addition of the mode
flags already changed the display. It makes sense to me to have the mounts
and numa_maps display the policy in the same format. However, if anyone
objects strongly, I can pass the desired nodemask delimeter as an arg to
mpol_to_str().
Note 2: Like show_numa_map(), I don't check the return code from
mpol_to_str(). I do use a longer buffer than the one provided by
show_numa_map(), which seems to have sufficed so far.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:23 +02:00
|
|
|
|
|
|
|
seq_printf(seq, ",mpol=%s", buffer);
|
2008-02-08 13:21:48 +01:00
|
|
|
}
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
|
|
|
|
static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
|
|
|
|
{
|
|
|
|
struct mempolicy *mpol = NULL;
|
|
|
|
if (sbinfo->mpol) {
|
|
|
|
spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
|
|
|
|
mpol = sbinfo->mpol;
|
|
|
|
mpol_get(mpol);
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
}
|
|
|
|
return mpol;
|
|
|
|
}
|
2008-02-08 13:21:48 +01:00
|
|
|
#endif /* CONFIG_TMPFS */
|
|
|
|
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
|
|
|
|
struct shmem_inode_info *info, unsigned long idx)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
mempolicy: rework mempolicy Reference Counting [yet again]
After further discussion with Christoph Lameter, it has become clear that my
earlier attempts to clean up the mempolicy reference counting were a bit of
overkill in some areas, resulting in superflous ref/unref in what are usually
fast paths. In other areas, further inspection reveals that I botched the
unref for interleave policies.
A separate patch, suitable for upstream/stable trees, fixes up the known
errors in the previous attempt to fix reference counting.
This patch reworks the memory policy referencing counting and, one hopes,
simplifies the code. Maybe I'll get it right this time.
See the update to the numa_memory_policy.txt document for a discussion of
memory policy reference counting that motivates this patch.
Summary:
Lookup of mempolicy, based on (vma, address) need only add a reference for
shared policy, and we need only unref the policy when finished for shared
policies. So, this patch backs out all of the unneeded extra reference
counting added by my previous attempt. It then unrefs only shared policies
when we're finished with them, using the mpol_cond_put() [conditional put]
helper function introduced by this patch.
Note that shmem_swapin() calls read_swap_cache_async() with a dummy vma
containing just the policy. read_swap_cache_async() can call alloc_page_vma()
multiple times, so we can't let alloc_page_vma() unref the shared policy in
this case. To avoid this, we make a copy of any non-null shared policy and
remove the MPOL_F_SHARED flag from the copy. This copy occurs before reading
a page [or multiple pages] from swap, so the overhead should not be an issue
here.
I introduced a new static inline function "mpol_cond_copy()" to copy the
shared policy to an on-stack policy and remove the flags that would require a
conditional free. The current implementation of mpol_cond_copy() assumes that
the struct mempolicy contains no pointers to dynamically allocated structures
that must be duplicated or reference counted during copy.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:16 +02:00
|
|
|
struct mempolicy mpol, *spol;
|
2005-04-17 00:20:36 +02:00
|
|
|
struct vm_area_struct pvma;
|
swapin_readahead: excise NUMA bogosity
For three years swapin_readahead has been cluttered with fanciful CONFIG_NUMA
code, advancing addr, and stepping on to the next vma at the boundary, to line
up the mempolicy for each page allocation.
It _might_ be a good idea to allocate swap more according to vma layout; but
the fact is, that's not how we do it at all, 2.6 even less than 2.4: swap is
allocated as needed for pages as they sink to the bottom of the inactive LRUs.
Sometimes that may match vma layout, but not so often that it's worth going
to these misleading vma->vm_next lengths: rip all that out.
Originally I intended to retain the incrementation of addr, but correct its
initial value: valid_swaphandles generally supplies an offset below the target
addr (this is readaround rather than readahead), but addr has not been
adjusted accordingly, so in the interleave case it has usually been allocating
the target page from the "wrong" node (though that may not matter very much).
But look at the equivalent shmem_swapin code: either by oversight or by
design, though it has all the apparatus for choosing a new mempolicy per page,
it uses the same idx throughout, choosing the same mempolicy and interleave
node for each page of the cluster.
Which is actually a much better strategy: each node has its own LRUs and its
own kswapd, so if you're betting on any particular relationship between swap
and node, the best bet is that nearby swap entries belong to pages from the
same node - even when the mempolicy of the target page is to interleave. And
examining a map of nodes corresponding to swap entries on a numa=fake system
bears this out. (We could later tweak swap allocation to make it even more
likely, but this patch is merely about removing cruft.)
So, neither adjust nor increment addr in swapin_readahead, and then
shmem_swapin can use it too; the pseudo-vma to pass policy need only be set up
once per cluster, and so few fields of pvma are used, let's skip the memset -
from shmem_alloc_page also.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:40 +01:00
|
|
|
struct page *page;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
mempolicy: rework mempolicy Reference Counting [yet again]
After further discussion with Christoph Lameter, it has become clear that my
earlier attempts to clean up the mempolicy reference counting were a bit of
overkill in some areas, resulting in superflous ref/unref in what are usually
fast paths. In other areas, further inspection reveals that I botched the
unref for interleave policies.
A separate patch, suitable for upstream/stable trees, fixes up the known
errors in the previous attempt to fix reference counting.
This patch reworks the memory policy referencing counting and, one hopes,
simplifies the code. Maybe I'll get it right this time.
See the update to the numa_memory_policy.txt document for a discussion of
memory policy reference counting that motivates this patch.
Summary:
Lookup of mempolicy, based on (vma, address) need only add a reference for
shared policy, and we need only unref the policy when finished for shared
policies. So, this patch backs out all of the unneeded extra reference
counting added by my previous attempt. It then unrefs only shared policies
when we're finished with them, using the mpol_cond_put() [conditional put]
helper function introduced by this patch.
Note that shmem_swapin() calls read_swap_cache_async() with a dummy vma
containing just the policy. read_swap_cache_async() can call alloc_page_vma()
multiple times, so we can't let alloc_page_vma() unref the shared policy in
this case. To avoid this, we make a copy of any non-null shared policy and
remove the MPOL_F_SHARED flag from the copy. This copy occurs before reading
a page [or multiple pages] from swap, so the overhead should not be an issue
here.
I introduced a new static inline function "mpol_cond_copy()" to copy the
shared policy to an on-stack policy and remove the flags that would require a
conditional free. The current implementation of mpol_cond_copy() assumes that
the struct mempolicy contains no pointers to dynamically allocated structures
that must be duplicated or reference counted during copy.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:16 +02:00
|
|
|
spol = mpol_cond_copy(&mpol,
|
|
|
|
mpol_shared_policy_lookup(&info->policy, idx));
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/* Create a pseudo vma that just contains the policy */
|
swapin_readahead: excise NUMA bogosity
For three years swapin_readahead has been cluttered with fanciful CONFIG_NUMA
code, advancing addr, and stepping on to the next vma at the boundary, to line
up the mempolicy for each page allocation.
It _might_ be a good idea to allocate swap more according to vma layout; but
the fact is, that's not how we do it at all, 2.6 even less than 2.4: swap is
allocated as needed for pages as they sink to the bottom of the inactive LRUs.
Sometimes that may match vma layout, but not so often that it's worth going
to these misleading vma->vm_next lengths: rip all that out.
Originally I intended to retain the incrementation of addr, but correct its
initial value: valid_swaphandles generally supplies an offset below the target
addr (this is readaround rather than readahead), but addr has not been
adjusted accordingly, so in the interleave case it has usually been allocating
the target page from the "wrong" node (though that may not matter very much).
But look at the equivalent shmem_swapin code: either by oversight or by
design, though it has all the apparatus for choosing a new mempolicy per page,
it uses the same idx throughout, choosing the same mempolicy and interleave
node for each page of the cluster.
Which is actually a much better strategy: each node has its own LRUs and its
own kswapd, so if you're betting on any particular relationship between swap
and node, the best bet is that nearby swap entries belong to pages from the
same node - even when the mempolicy of the target page is to interleave. And
examining a map of nodes corresponding to swap entries on a numa=fake system
bears this out. (We could later tweak swap allocation to make it even more
likely, but this patch is merely about removing cruft.)
So, neither adjust nor increment addr in swapin_readahead, and then
shmem_swapin can use it too; the pseudo-vma to pass policy need only be set up
once per cluster, and so few fields of pvma are used, let's skip the memset -
from shmem_alloc_page also.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:40 +01:00
|
|
|
pvma.vm_start = 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
pvma.vm_pgoff = idx;
|
swapin_readahead: excise NUMA bogosity
For three years swapin_readahead has been cluttered with fanciful CONFIG_NUMA
code, advancing addr, and stepping on to the next vma at the boundary, to line
up the mempolicy for each page allocation.
It _might_ be a good idea to allocate swap more according to vma layout; but
the fact is, that's not how we do it at all, 2.6 even less than 2.4: swap is
allocated as needed for pages as they sink to the bottom of the inactive LRUs.
Sometimes that may match vma layout, but not so often that it's worth going
to these misleading vma->vm_next lengths: rip all that out.
Originally I intended to retain the incrementation of addr, but correct its
initial value: valid_swaphandles generally supplies an offset below the target
addr (this is readaround rather than readahead), but addr has not been
adjusted accordingly, so in the interleave case it has usually been allocating
the target page from the "wrong" node (though that may not matter very much).
But look at the equivalent shmem_swapin code: either by oversight or by
design, though it has all the apparatus for choosing a new mempolicy per page,
it uses the same idx throughout, choosing the same mempolicy and interleave
node for each page of the cluster.
Which is actually a much better strategy: each node has its own LRUs and its
own kswapd, so if you're betting on any particular relationship between swap
and node, the best bet is that nearby swap entries belong to pages from the
same node - even when the mempolicy of the target page is to interleave. And
examining a map of nodes corresponding to swap entries on a numa=fake system
bears this out. (We could later tweak swap allocation to make it even more
likely, but this patch is merely about removing cruft.)
So, neither adjust nor increment addr in swapin_readahead, and then
shmem_swapin can use it too; the pseudo-vma to pass policy need only be set up
once per cluster, and so few fields of pvma are used, let's skip the memset -
from shmem_alloc_page also.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:40 +01:00
|
|
|
pvma.vm_ops = NULL;
|
mempolicy: rework mempolicy Reference Counting [yet again]
After further discussion with Christoph Lameter, it has become clear that my
earlier attempts to clean up the mempolicy reference counting were a bit of
overkill in some areas, resulting in superflous ref/unref in what are usually
fast paths. In other areas, further inspection reveals that I botched the
unref for interleave policies.
A separate patch, suitable for upstream/stable trees, fixes up the known
errors in the previous attempt to fix reference counting.
This patch reworks the memory policy referencing counting and, one hopes,
simplifies the code. Maybe I'll get it right this time.
See the update to the numa_memory_policy.txt document for a discussion of
memory policy reference counting that motivates this patch.
Summary:
Lookup of mempolicy, based on (vma, address) need only add a reference for
shared policy, and we need only unref the policy when finished for shared
policies. So, this patch backs out all of the unneeded extra reference
counting added by my previous attempt. It then unrefs only shared policies
when we're finished with them, using the mpol_cond_put() [conditional put]
helper function introduced by this patch.
Note that shmem_swapin() calls read_swap_cache_async() with a dummy vma
containing just the policy. read_swap_cache_async() can call alloc_page_vma()
multiple times, so we can't let alloc_page_vma() unref the shared policy in
this case. To avoid this, we make a copy of any non-null shared policy and
remove the MPOL_F_SHARED flag from the copy. This copy occurs before reading
a page [or multiple pages] from swap, so the overhead should not be an issue
here.
I introduced a new static inline function "mpol_cond_copy()" to copy the
shared policy to an on-stack policy and remove the flags that would require a
conditional free. The current implementation of mpol_cond_copy() assumes that
the struct mempolicy contains no pointers to dynamically allocated structures
that must be duplicated or reference counted during copy.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:16 +02:00
|
|
|
pvma.vm_policy = spol;
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
page = swapin_readahead(entry, gfp, &pvma, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
static struct page *shmem_alloc_page(gfp_t gfp,
|
|
|
|
struct shmem_inode_info *info, unsigned long idx)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
struct vm_area_struct pvma;
|
|
|
|
|
swapin_readahead: excise NUMA bogosity
For three years swapin_readahead has been cluttered with fanciful CONFIG_NUMA
code, advancing addr, and stepping on to the next vma at the boundary, to line
up the mempolicy for each page allocation.
It _might_ be a good idea to allocate swap more according to vma layout; but
the fact is, that's not how we do it at all, 2.6 even less than 2.4: swap is
allocated as needed for pages as they sink to the bottom of the inactive LRUs.
Sometimes that may match vma layout, but not so often that it's worth going
to these misleading vma->vm_next lengths: rip all that out.
Originally I intended to retain the incrementation of addr, but correct its
initial value: valid_swaphandles generally supplies an offset below the target
addr (this is readaround rather than readahead), but addr has not been
adjusted accordingly, so in the interleave case it has usually been allocating
the target page from the "wrong" node (though that may not matter very much).
But look at the equivalent shmem_swapin code: either by oversight or by
design, though it has all the apparatus for choosing a new mempolicy per page,
it uses the same idx throughout, choosing the same mempolicy and interleave
node for each page of the cluster.
Which is actually a much better strategy: each node has its own LRUs and its
own kswapd, so if you're betting on any particular relationship between swap
and node, the best bet is that nearby swap entries belong to pages from the
same node - even when the mempolicy of the target page is to interleave. And
examining a map of nodes corresponding to swap entries on a numa=fake system
bears this out. (We could later tweak swap allocation to make it even more
likely, but this patch is merely about removing cruft.)
So, neither adjust nor increment addr in swapin_readahead, and then
shmem_swapin can use it too; the pseudo-vma to pass policy need only be set up
once per cluster, and so few fields of pvma are used, let's skip the memset -
from shmem_alloc_page also.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:40 +01:00
|
|
|
/* Create a pseudo vma that just contains the policy */
|
|
|
|
pvma.vm_start = 0;
|
2005-04-17 00:20:36 +02:00
|
|
|
pvma.vm_pgoff = idx;
|
swapin_readahead: excise NUMA bogosity
For three years swapin_readahead has been cluttered with fanciful CONFIG_NUMA
code, advancing addr, and stepping on to the next vma at the boundary, to line
up the mempolicy for each page allocation.
It _might_ be a good idea to allocate swap more according to vma layout; but
the fact is, that's not how we do it at all, 2.6 even less than 2.4: swap is
allocated as needed for pages as they sink to the bottom of the inactive LRUs.
Sometimes that may match vma layout, but not so often that it's worth going
to these misleading vma->vm_next lengths: rip all that out.
Originally I intended to retain the incrementation of addr, but correct its
initial value: valid_swaphandles generally supplies an offset below the target
addr (this is readaround rather than readahead), but addr has not been
adjusted accordingly, so in the interleave case it has usually been allocating
the target page from the "wrong" node (though that may not matter very much).
But look at the equivalent shmem_swapin code: either by oversight or by
design, though it has all the apparatus for choosing a new mempolicy per page,
it uses the same idx throughout, choosing the same mempolicy and interleave
node for each page of the cluster.
Which is actually a much better strategy: each node has its own LRUs and its
own kswapd, so if you're betting on any particular relationship between swap
and node, the best bet is that nearby swap entries belong to pages from the
same node - even when the mempolicy of the target page is to interleave. And
examining a map of nodes corresponding to swap entries on a numa=fake system
bears this out. (We could later tweak swap allocation to make it even more
likely, but this patch is merely about removing cruft.)
So, neither adjust nor increment addr in swapin_readahead, and then
shmem_swapin can use it too; the pseudo-vma to pass policy need only be set up
once per cluster, and so few fields of pvma are used, let's skip the memset -
from shmem_alloc_page also.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:40 +01:00
|
|
|
pvma.vm_ops = NULL;
|
|
|
|
pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
|
mempolicy: rework mempolicy Reference Counting [yet again]
After further discussion with Christoph Lameter, it has become clear that my
earlier attempts to clean up the mempolicy reference counting were a bit of
overkill in some areas, resulting in superflous ref/unref in what are usually
fast paths. In other areas, further inspection reveals that I botched the
unref for interleave policies.
A separate patch, suitable for upstream/stable trees, fixes up the known
errors in the previous attempt to fix reference counting.
This patch reworks the memory policy referencing counting and, one hopes,
simplifies the code. Maybe I'll get it right this time.
See the update to the numa_memory_policy.txt document for a discussion of
memory policy reference counting that motivates this patch.
Summary:
Lookup of mempolicy, based on (vma, address) need only add a reference for
shared policy, and we need only unref the policy when finished for shared
policies. So, this patch backs out all of the unneeded extra reference
counting added by my previous attempt. It then unrefs only shared policies
when we're finished with them, using the mpol_cond_put() [conditional put]
helper function introduced by this patch.
Note that shmem_swapin() calls read_swap_cache_async() with a dummy vma
containing just the policy. read_swap_cache_async() can call alloc_page_vma()
multiple times, so we can't let alloc_page_vma() unref the shared policy in
this case. To avoid this, we make a copy of any non-null shared policy and
remove the MPOL_F_SHARED flag from the copy. This copy occurs before reading
a page [or multiple pages] from swap, so the overhead should not be an issue
here.
I introduced a new static inline function "mpol_cond_copy()" to copy the
shared policy to an on-stack policy and remove the flags that would require a
conditional free. The current implementation of mpol_cond_copy() assumes that
the struct mempolicy contains no pointers to dynamically allocated structures
that must be duplicated or reference counted during copy.
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:16 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* alloc_page_vma() will drop the shared policy reference
|
|
|
|
*/
|
|
|
|
return alloc_page_vma(gfp, &pvma, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2008-02-08 13:21:48 +01:00
|
|
|
#else /* !CONFIG_NUMA */
|
|
|
|
#ifdef CONFIG_TMPFS
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
|
2008-02-08 13:21:48 +01:00
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_TMPFS */
|
|
|
|
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
|
|
|
|
struct shmem_inode_info *info, unsigned long idx)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
return swapin_readahead(entry, gfp, NULL, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
static inline struct page *shmem_alloc_page(gfp_t gfp,
|
|
|
|
struct shmem_inode_info *info, unsigned long idx)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-11-28 19:55:10 +01:00
|
|
|
return alloc_page(gfp);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2008-02-08 13:21:48 +01:00
|
|
|
#endif /* CONFIG_NUMA */
|
2005-04-17 00:20:36 +02:00
|
|
|
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
|
|
|
|
static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* shmem_getpage - either get the page from swap or allocate a new one
|
|
|
|
*
|
|
|
|
* If we allocate a new one we do not mark it dirty. That's up to the
|
|
|
|
* vm. If we swap it in we mark it dirty since we also free the swap
|
|
|
|
* entry since a page cannot live in both the swap and page cache
|
|
|
|
*/
|
|
|
|
static int shmem_getpage(struct inode *inode, unsigned long idx,
|
|
|
|
struct page **pagep, enum sgp_type sgp, int *type)
|
|
|
|
{
|
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
struct shmem_inode_info *info = SHMEM_I(inode);
|
|
|
|
struct shmem_sb_info *sbinfo;
|
|
|
|
struct page *filepage = *pagep;
|
|
|
|
struct page *swappage;
|
|
|
|
swp_entry_t *entry;
|
|
|
|
swp_entry_t swap;
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
gfp_t gfp;
|
2005-04-17 00:20:36 +02:00
|
|
|
int error;
|
|
|
|
|
|
|
|
if (idx >= SHMEM_MAX_INDEX)
|
|
|
|
return -EFBIG;
|
2007-07-19 10:46:59 +02:00
|
|
|
|
|
|
|
if (type)
|
2007-07-19 10:47:05 +02:00
|
|
|
*type = 0;
|
2007-07-19 10:46:59 +02:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* Normally, filepage is NULL on entry, and either found
|
|
|
|
* uptodate immediately, or allocated and zeroed, or read
|
|
|
|
* in under swappage, which is then assigned to filepage.
|
2008-02-05 07:28:44 +01:00
|
|
|
* But shmem_readpage (required for splice) passes in a locked
|
2007-06-04 10:00:39 +02:00
|
|
|
* filepage, which may be found not uptodate by other callers
|
|
|
|
* too, and may need to be copied from the swappage read in.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
repeat:
|
|
|
|
if (!filepage)
|
|
|
|
filepage = find_lock_page(mapping, idx);
|
|
|
|
if (filepage && PageUptodate(filepage))
|
|
|
|
goto done;
|
|
|
|
error = 0;
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
gfp = mapping_gfp_mask(mapping);
|
2008-02-05 07:28:54 +01:00
|
|
|
if (!filepage) {
|
|
|
|
/*
|
|
|
|
* Try to preload while we can wait, to not make a habit of
|
|
|
|
* draining atomic reserves; but don't latch on to this cpu.
|
|
|
|
*/
|
|
|
|
error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
|
|
|
|
if (error)
|
|
|
|
goto failed;
|
|
|
|
radix_tree_preload_end();
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
shmem_recalc_inode(inode);
|
|
|
|
entry = shmem_swp_alloc(info, idx, sgp);
|
|
|
|
if (IS_ERR(entry)) {
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
error = PTR_ERR(entry);
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
swap = *entry;
|
|
|
|
|
|
|
|
if (swap.val) {
|
|
|
|
/* Look it up and read it in.. */
|
|
|
|
swappage = lookup_swap_cache(swap);
|
|
|
|
if (!swappage) {
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
/* here we actually do the io */
|
2007-07-19 10:47:05 +02:00
|
|
|
if (type && !(*type & VM_FAULT_MAJOR)) {
|
2006-06-30 10:55:45 +02:00
|
|
|
__count_vm_event(PGMAJFAULT);
|
2007-07-19 10:47:05 +02:00
|
|
|
*type |= VM_FAULT_MAJOR;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-06-30 10:55:45 +02:00
|
|
|
spin_unlock(&info->lock);
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
swappage = shmem_swapin(swap, gfp, info, idx);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!swappage) {
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
entry = shmem_swp_alloc(info, idx, sgp);
|
|
|
|
if (IS_ERR(entry))
|
|
|
|
error = PTR_ERR(entry);
|
|
|
|
else {
|
|
|
|
if (entry->val == swap.val)
|
|
|
|
error = -ENOMEM;
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
}
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
if (error)
|
|
|
|
goto failed;
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
wait_on_page_locked(swappage);
|
|
|
|
page_cache_release(swappage);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We have to do this with page locked to prevent races */
|
2008-08-02 12:01:03 +02:00
|
|
|
if (!trylock_page(swappage)) {
|
2005-04-17 00:20:36 +02:00
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
wait_on_page_locked(swappage);
|
|
|
|
page_cache_release(swappage);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
if (PageWriteback(swappage)) {
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
wait_on_page_writeback(swappage);
|
|
|
|
unlock_page(swappage);
|
|
|
|
page_cache_release(swappage);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
if (!PageUptodate(swappage)) {
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
unlock_page(swappage);
|
|
|
|
page_cache_release(swappage);
|
|
|
|
error = -EIO;
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (filepage) {
|
|
|
|
shmem_swp_set(info, entry, 0);
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
delete_from_swap_cache(swappage);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
copy_highpage(filepage, swappage);
|
|
|
|
unlock_page(swappage);
|
|
|
|
page_cache_release(swappage);
|
|
|
|
flush_dcache_page(filepage);
|
|
|
|
SetPageUptodate(filepage);
|
|
|
|
set_page_dirty(filepage);
|
|
|
|
swap_free(swap);
|
2008-07-26 04:45:30 +02:00
|
|
|
} else if (!(error = add_to_page_cache_locked(swappage, mapping,
|
|
|
|
idx, GFP_NOWAIT))) {
|
2005-04-17 00:20:36 +02:00
|
|
|
info->flags |= SHMEM_PAGEIN;
|
|
|
|
shmem_swp_set(info, entry, 0);
|
|
|
|
shmem_swp_unmap(entry);
|
2008-02-05 07:28:50 +01:00
|
|
|
delete_from_swap_cache(swappage);
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
|
|
|
filepage = swappage;
|
2008-02-05 07:28:50 +01:00
|
|
|
set_page_dirty(filepage);
|
2005-04-17 00:20:36 +02:00
|
|
|
swap_free(swap);
|
|
|
|
} else {
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
spin_unlock(&info->lock);
|
2008-02-07 09:14:22 +01:00
|
|
|
if (error == -ENOMEM) {
|
2009-05-01 00:08:19 +02:00
|
|
|
/*
|
|
|
|
* reclaim from proper memory cgroup and
|
|
|
|
* call memcg's OOM if needed.
|
|
|
|
*/
|
|
|
|
error = mem_cgroup_shmem_charge_fallback(
|
|
|
|
swappage,
|
2009-01-08 03:08:35 +01:00
|
|
|
current->mm,
|
2008-07-25 10:47:15 +02:00
|
|
|
gfp);
|
2009-01-08 03:08:35 +01:00
|
|
|
if (error) {
|
|
|
|
unlock_page(swappage);
|
|
|
|
page_cache_release(swappage);
|
2008-02-07 09:14:22 +01:00
|
|
|
goto failed;
|
2009-01-08 03:08:35 +01:00
|
|
|
}
|
2008-02-07 09:14:22 +01:00
|
|
|
}
|
2009-01-08 03:08:35 +01:00
|
|
|
unlock_page(swappage);
|
|
|
|
page_cache_release(swappage);
|
2005-04-17 00:20:36 +02:00
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
} else if (sgp == SGP_READ && !filepage) {
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
filepage = find_get_page(mapping, idx);
|
|
|
|
if (filepage &&
|
2008-08-02 12:01:03 +02:00
|
|
|
(!PageUptodate(filepage) || !trylock_page(filepage))) {
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
|
|
|
wait_on_page_locked(filepage);
|
|
|
|
page_cache_release(filepage);
|
|
|
|
filepage = NULL;
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
} else {
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
sbinfo = SHMEM_SB(inode->i_sb);
|
2005-06-22 02:15:04 +02:00
|
|
|
if (sbinfo->max_blocks) {
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (sbinfo->free_blocks == 0 ||
|
|
|
|
shmem_acct_block(info->flags)) {
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
error = -ENOSPC;
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
sbinfo->free_blocks--;
|
|
|
|
inode->i_blocks += BLOCKS_PER_PAGE;
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
} else if (shmem_acct_block(info->flags)) {
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
error = -ENOSPC;
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!filepage) {
|
memcg: remove refcnt from page_cgroup
memcg: performance improvements
Patch Description
1/5 ... remove refcnt fron page_cgroup patch (shmem handling is fixed)
2/5 ... swapcache handling patch
3/5 ... add helper function for shmem's memory reclaim patch
4/5 ... optimize by likely/unlikely ppatch
5/5 ... remove redundunt check patch (shmem handling is fixed.)
Unix bench result.
== 2.6.26-rc2-mm1 + memory resource controller
Execl Throughput 2915.4 lps (29.6 secs, 3 samples)
C Compiler Throughput 1019.3 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5796.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1097.7 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 565.3 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1022128.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 544057.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 346481.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 319325.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 148788.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 99051.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2058917.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1606109.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 854789.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 126145.2 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 2915.4 678.0
File Copy 1024 bufsize 2000 maxblocks 3960.0 346481.0 875.0
File Copy 256 bufsize 500 maxblocks 1655.0 99051.0 598.5
File Copy 4096 bufsize 8000 maxblocks 5800.0 854789.0 1473.8
Shell Scripts (8 concurrent) 6.0 1097.7 1829.5
=========
FINAL SCORE 991.3
== 2.6.26-rc2-mm1 + this set ==
Execl Throughput 3012.9 lps (29.9 secs, 3 samples)
C Compiler Throughput 981.0 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5872.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1120.3 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 578.0 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1003993.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 550452.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 347159.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 314644.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 151852.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 101000.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2033256.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1611814.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 847979.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 128148.7 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 3012.9 700.7
File Copy 1024 bufsize 2000 maxblocks 3960.0 347159.0 876.7
File Copy 256 bufsize 500 maxblocks 1655.0 101000.0 610.3
File Copy 4096 bufsize 8000 maxblocks 5800.0 847979.0 1462.0
Shell Scripts (8 concurrent) 6.0 1120.3 1867.2
=========
FINAL SCORE 1004.6
This patch:
Remove refcnt from page_cgroup().
After this,
* A page is charged only when !page_mapped() && no page_cgroup is assigned.
* Anon page is newly mapped.
* File page is added to mapping->tree.
* A page is uncharged only when
* Anon page is fully unmapped.
* File page is removed from LRU.
There is no change in behavior from user's view.
This patch also removes unnecessary calls in rmap.c which was used only for
refcnt mangement.
[akpm@linux-foundation.org: fix warning]
[hugh@veritas.com: fix shmem_unuse_inode charging]
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: YAMAMOTO Takashi <yamamoto@valinux.co.jp>
Cc: Paul Menage <menage@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:14 +02:00
|
|
|
int ret;
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
swapin needs gfp_mask for loop on tmpfs
Building in a filesystem on a loop device on a tmpfs file can hang when
swapping, the loop thread caught in that infamous throttle_vm_writeout.
In theory this is a long standing problem, which I've either never seen in
practice, or long ago suppressed the recollection, after discounting my load
and my tmpfs size as unrealistically high. But now, with the new aops, it has
become easy to hang on one machine.
Loop used to grab_cache_page before the old prepare_write to tmpfs, which
seems to have been enough to free up some memory for any swapin needed; but
the new write_begin lets tmpfs find or allocate the page (much nicer, since
grab_cache_page missed tmpfs pages in swapcache).
When allocating a fresh page, tmpfs respects loop's mapping_gfp_mask, which
has __GFP_IO|__GFP_FS stripped off, and throttle_vm_writeout is designed to
break out when __GFP_IO or GFP_FS is unset; but when tmfps swaps in,
read_swap_cache_async allocates with GFP_HIGHUSER_MOVABLE regardless of the
mapping_gfp_mask - hence the hang.
So, pass gfp_mask down the line from shmem_getpage to shmem_swapin to
swapin_readahead to read_swap_cache_async to add_to_swap_cache.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Acked-by: Rik van Riel <riel@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-02-05 07:28:42 +01:00
|
|
|
filepage = shmem_alloc_page(gfp, info, idx);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!filepage) {
|
|
|
|
shmem_unacct_blocks(info->flags, 1);
|
|
|
|
shmem_free_blocks(inode, 1);
|
|
|
|
error = -ENOMEM;
|
|
|
|
goto failed;
|
|
|
|
}
|
2008-10-19 05:26:30 +02:00
|
|
|
SetPageSwapBacked(filepage);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-02-07 09:14:22 +01:00
|
|
|
/* Precharge page while we can wait, compensate after */
|
|
|
|
error = mem_cgroup_cache_charge(filepage, current->mm,
|
2009-01-08 03:08:10 +01:00
|
|
|
GFP_KERNEL);
|
2008-02-07 09:14:22 +01:00
|
|
|
if (error) {
|
|
|
|
page_cache_release(filepage);
|
|
|
|
shmem_unacct_blocks(info->flags, 1);
|
|
|
|
shmem_free_blocks(inode, 1);
|
|
|
|
filepage = NULL;
|
|
|
|
goto failed;
|
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock(&info->lock);
|
|
|
|
entry = shmem_swp_alloc(info, idx, sgp);
|
|
|
|
if (IS_ERR(entry))
|
|
|
|
error = PTR_ERR(entry);
|
|
|
|
else {
|
|
|
|
swap = *entry;
|
|
|
|
shmem_swp_unmap(entry);
|
|
|
|
}
|
memcg: remove refcnt from page_cgroup
memcg: performance improvements
Patch Description
1/5 ... remove refcnt fron page_cgroup patch (shmem handling is fixed)
2/5 ... swapcache handling patch
3/5 ... add helper function for shmem's memory reclaim patch
4/5 ... optimize by likely/unlikely ppatch
5/5 ... remove redundunt check patch (shmem handling is fixed.)
Unix bench result.
== 2.6.26-rc2-mm1 + memory resource controller
Execl Throughput 2915.4 lps (29.6 secs, 3 samples)
C Compiler Throughput 1019.3 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5796.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1097.7 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 565.3 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1022128.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 544057.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 346481.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 319325.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 148788.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 99051.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2058917.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1606109.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 854789.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 126145.2 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 2915.4 678.0
File Copy 1024 bufsize 2000 maxblocks 3960.0 346481.0 875.0
File Copy 256 bufsize 500 maxblocks 1655.0 99051.0 598.5
File Copy 4096 bufsize 8000 maxblocks 5800.0 854789.0 1473.8
Shell Scripts (8 concurrent) 6.0 1097.7 1829.5
=========
FINAL SCORE 991.3
== 2.6.26-rc2-mm1 + this set ==
Execl Throughput 3012.9 lps (29.9 secs, 3 samples)
C Compiler Throughput 981.0 lpm (60.0 secs, 3 samples)
Shell Scripts (1 concurrent) 5872.0 lpm (60.0 secs, 3 samples)
Shell Scripts (8 concurrent) 1120.3 lpm (60.0 secs, 3 samples)
Shell Scripts (16 concurrent) 578.0 lpm (60.0 secs, 3 samples)
File Read 1024 bufsize 2000 maxblocks 1003993.0 KBps (30.0 secs, 3 samples)
File Write 1024 bufsize 2000 maxblocks 550452.0 KBps (30.0 secs, 3 samples)
File Copy 1024 bufsize 2000 maxblocks 347159.0 KBps (30.0 secs, 3 samples)
File Read 256 bufsize 500 maxblocks 314644.0 KBps (30.0 secs, 3 samples)
File Write 256 bufsize 500 maxblocks 151852.0 KBps (30.0 secs, 3 samples)
File Copy 256 bufsize 500 maxblocks 101000.0 KBps (30.0 secs, 3 samples)
File Read 4096 bufsize 8000 maxblocks 2033256.0 KBps (30.0 secs, 3 samples)
File Write 4096 bufsize 8000 maxblocks 1611814.0 KBps (30.0 secs, 3 samples)
File Copy 4096 bufsize 8000 maxblocks 847979.0 KBps (30.0 secs, 3 samples)
Dc: sqrt(2) to 99 decimal places 128148.7 lpm (30.0 secs, 3 samples)
INDEX VALUES
TEST BASELINE RESULT INDEX
Execl Throughput 43.0 3012.9 700.7
File Copy 1024 bufsize 2000 maxblocks 3960.0 347159.0 876.7
File Copy 256 bufsize 500 maxblocks 1655.0 101000.0 610.3
File Copy 4096 bufsize 8000 maxblocks 5800.0 847979.0 1462.0
Shell Scripts (8 concurrent) 6.0 1120.3 1867.2
=========
FINAL SCORE 1004.6
This patch:
Remove refcnt from page_cgroup().
After this,
* A page is charged only when !page_mapped() && no page_cgroup is assigned.
* Anon page is newly mapped.
* File page is added to mapping->tree.
* A page is uncharged only when
* Anon page is fully unmapped.
* File page is removed from LRU.
There is no change in behavior from user's view.
This patch also removes unnecessary calls in rmap.c which was used only for
refcnt mangement.
[akpm@linux-foundation.org: fix warning]
[hugh@veritas.com: fix shmem_unuse_inode charging]
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Li Zefan <lizf@cn.fujitsu.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: YAMAMOTO Takashi <yamamoto@valinux.co.jp>
Cc: Paul Menage <menage@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:14 +02:00
|
|
|
ret = error || swap.val;
|
|
|
|
if (ret)
|
|
|
|
mem_cgroup_uncharge_cache_page(filepage);
|
|
|
|
else
|
|
|
|
ret = add_to_page_cache_lru(filepage, mapping,
|
|
|
|
idx, GFP_NOWAIT);
|
|
|
|
/*
|
|
|
|
* At add_to_page_cache_lru() failure, uncharge will
|
|
|
|
* be done automatically.
|
|
|
|
*/
|
|
|
|
if (ret) {
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock(&info->lock);
|
|
|
|
page_cache_release(filepage);
|
|
|
|
shmem_unacct_blocks(info->flags, 1);
|
|
|
|
shmem_free_blocks(inode, 1);
|
|
|
|
filepage = NULL;
|
|
|
|
if (error)
|
|
|
|
goto failed;
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
info->flags |= SHMEM_PAGEIN;
|
|
|
|
}
|
|
|
|
|
|
|
|
info->alloced++;
|
|
|
|
spin_unlock(&info->lock);
|
2007-11-28 19:55:10 +01:00
|
|
|
clear_highpage(filepage);
|
2005-04-17 00:20:36 +02:00
|
|
|
flush_dcache_page(filepage);
|
|
|
|
SetPageUptodate(filepage);
|
2008-02-05 07:28:51 +01:00
|
|
|
if (sgp == SGP_DIRTY)
|
|
|
|
set_page_dirty(filepage);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
done:
|
2008-02-05 07:28:44 +01:00
|
|
|
*pagep = filepage;
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
failed:
|
|
|
|
if (*pagep != filepage) {
|
|
|
|
unlock_page(filepage);
|
|
|
|
page_cache_release(filepage);
|
|
|
|
}
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
2007-07-19 10:47:03 +02:00
|
|
|
static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-12-08 11:36:44 +01:00
|
|
|
struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
|
2005-04-17 00:20:36 +02:00
|
|
|
int error;
|
2007-07-19 10:47:03 +02:00
|
|
|
int ret;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-07-19 10:47:03 +02:00
|
|
|
if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
|
|
|
|
return VM_FAULT_SIGBUS;
|
mm: fix fault vs invalidate race for linear mappings
Fix the race between invalidate_inode_pages and do_no_page.
Andrea Arcangeli identified a subtle race between invalidation of pages from
pagecache with userspace mappings, and do_no_page.
The issue is that invalidation has to shoot down all mappings to the page,
before it can be discarded from the pagecache. Between shooting down ptes to
a particular page, and actually dropping the struct page from the pagecache,
do_no_page from any process might fault on that page and establish a new
mapping to the page just before it gets discarded from the pagecache.
The most common case where such invalidation is used is in file truncation.
This case was catered for by doing a sort of open-coded seqlock between the
file's i_size, and its truncate_count.
Truncation will decrease i_size, then increment truncate_count before
unmapping userspace pages; do_no_page will read truncate_count, then find the
page if it is within i_size, and then check truncate_count under the page
table lock and back out and retry if it had subsequently been changed (ptl
will serialise against unmapping, and ensure a potentially updated
truncate_count is actually visible).
Complexity and documentation issues aside, the locking protocol fails in the
case where we would like to invalidate pagecache inside i_size. do_no_page
can come in anytime and filemap_nopage is not aware of the invalidation in
progress (as it is when it is outside i_size). The end result is that
dangling (->mapping == NULL) pages that appear to be from a particular file
may be mapped into userspace with nonsense data. Valid mappings to the same
place will see a different page.
Andrea implemented two working fixes, one using a real seqlock, another using
a page->flags bit. He also proposed using the page lock in do_no_page, but
that was initially considered too heavyweight. However, it is not a global or
per-file lock, and the page cacheline is modified in do_no_page to increment
_count and _mapcount anyway, so a further modification should not be a large
performance hit. Scalability is not an issue.
This patch implements this latter approach. ->nopage implementations return
with the page locked if it is possible for their underlying file to be
invalidated (in that case, they must set a special vm_flags bit to indicate
so). do_no_page only unlocks the page after setting up the mapping
completely. invalidation is excluded because it holds the page lock during
invalidation of each page (and ensures that the page is not mapped while
holding the lock).
This also allows significant simplifications in do_no_page, because we have
the page locked in the right place in the pagecache from the start.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-19 10:46:57 +02:00
|
|
|
|
2008-02-05 07:28:43 +01:00
|
|
|
error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
|
2007-07-19 10:47:03 +02:00
|
|
|
if (error)
|
|
|
|
return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-07-19 10:47:05 +02:00
|
|
|
return ret | VM_FAULT_LOCKED;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA
|
2007-10-16 10:26:26 +02:00
|
|
|
static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-12-08 11:36:44 +01:00
|
|
|
struct inode *i = vma->vm_file->f_path.dentry->d_inode;
|
2005-04-17 00:20:36 +02:00
|
|
|
return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
|
|
|
|
}
|
|
|
|
|
2007-10-16 10:26:26 +02:00
|
|
|
static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-12-08 11:36:44 +01:00
|
|
|
struct inode *i = vma->vm_file->f_path.dentry->d_inode;
|
2005-04-17 00:20:36 +02:00
|
|
|
unsigned long idx;
|
|
|
|
|
|
|
|
idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
|
|
|
|
return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
int shmem_lock(struct file *file, int lock, struct user_struct *user)
|
|
|
|
{
|
2006-12-08 11:36:44 +01:00
|
|
|
struct inode *inode = file->f_path.dentry->d_inode;
|
2005-04-17 00:20:36 +02:00
|
|
|
struct shmem_inode_info *info = SHMEM_I(inode);
|
|
|
|
int retval = -ENOMEM;
|
|
|
|
|
|
|
|
spin_lock(&info->lock);
|
|
|
|
if (lock && !(info->flags & VM_LOCKED)) {
|
|
|
|
if (!user_shm_lock(inode->i_size, user))
|
|
|
|
goto out_nomem;
|
|
|
|
info->flags |= VM_LOCKED;
|
2008-10-19 05:26:43 +02:00
|
|
|
mapping_set_unevictable(file->f_mapping);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
if (!lock && (info->flags & VM_LOCKED) && user) {
|
|
|
|
user_shm_unlock(inode->i_size, user);
|
|
|
|
info->flags &= ~VM_LOCKED;
|
2008-10-19 05:26:43 +02:00
|
|
|
mapping_clear_unevictable(file->f_mapping);
|
|
|
|
scan_mapping_unevictable_pages(file->f_mapping);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
retval = 0;
|
2008-10-19 05:26:43 +02:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
out_nomem:
|
|
|
|
spin_unlock(&info->lock);
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
2007-03-01 05:11:03 +01:00
|
|
|
static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
file_accessed(file);
|
|
|
|
vma->vm_ops = &shmem_vm_ops;
|
2007-07-19 10:47:03 +02:00
|
|
|
vma->vm_flags |= VM_CAN_NONLINEAR;
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-02-24 21:51:52 +01:00
|
|
|
static struct inode *shmem_get_inode(struct super_block *sb, int mode,
|
|
|
|
dev_t dev, unsigned long flags)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
struct shmem_inode_info *info;
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
|
|
|
|
|
2008-02-05 07:28:47 +01:00
|
|
|
if (shmem_reserve_inode(sb))
|
|
|
|
return NULL;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
inode = new_inode(sb);
|
|
|
|
if (inode) {
|
|
|
|
inode->i_mode = mode;
|
2008-11-14 00:39:12 +01:00
|
|
|
inode->i_uid = current_fsuid();
|
|
|
|
inode->i_gid = current_fsgid();
|
2005-04-17 00:20:36 +02:00
|
|
|
inode->i_blocks = 0;
|
|
|
|
inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
|
|
|
|
inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
|
2006-10-17 09:09:45 +02:00
|
|
|
inode->i_generation = get_seconds();
|
2005-04-17 00:20:36 +02:00
|
|
|
info = SHMEM_I(inode);
|
|
|
|
memset(info, 0, (char *)inode - (char *)info);
|
|
|
|
spin_lock_init(&info->lock);
|
2009-02-24 21:51:52 +01:00
|
|
|
info->flags = flags & VM_NORESERVE;
|
2005-04-17 00:20:36 +02:00
|
|
|
INIT_LIST_HEAD(&info->swaplist);
|
|
|
|
|
|
|
|
switch (mode & S_IFMT) {
|
|
|
|
default:
|
2006-09-29 11:01:35 +02:00
|
|
|
inode->i_op = &shmem_special_inode_operations;
|
2005-04-17 00:20:36 +02:00
|
|
|
init_special_inode(inode, mode, dev);
|
|
|
|
break;
|
|
|
|
case S_IFREG:
|
2008-07-29 00:46:19 +02:00
|
|
|
inode->i_mapping->a_ops = &shmem_aops;
|
2005-04-17 00:20:36 +02:00
|
|
|
inode->i_op = &shmem_inode_operations;
|
|
|
|
inode->i_fop = &shmem_file_operations;
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
mpol_shared_policy_init(&info->policy,
|
|
|
|
shmem_get_sbmpol(sbinfo));
|
2005-04-17 00:20:36 +02:00
|
|
|
break;
|
|
|
|
case S_IFDIR:
|
2006-10-01 08:29:04 +02:00
|
|
|
inc_nlink(inode);
|
2005-04-17 00:20:36 +02:00
|
|
|
/* Some things misbehave if size == 0 on a directory */
|
|
|
|
inode->i_size = 2 * BOGO_DIRENT_SIZE;
|
|
|
|
inode->i_op = &shmem_dir_inode_operations;
|
|
|
|
inode->i_fop = &simple_dir_operations;
|
|
|
|
break;
|
|
|
|
case S_IFLNK:
|
|
|
|
/*
|
|
|
|
* Must not load anything in the rbtree,
|
|
|
|
* mpol_free_shared_policy will not be called.
|
|
|
|
*/
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
mpol_shared_policy_init(&info->policy, NULL);
|
2005-04-17 00:20:36 +02:00
|
|
|
break;
|
|
|
|
}
|
2008-02-05 07:28:47 +01:00
|
|
|
} else
|
|
|
|
shmem_free_inode(sb);
|
2005-04-17 00:20:36 +02:00
|
|
|
return inode;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_TMPFS
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_symlink_inode_operations;
|
|
|
|
static const struct inode_operations shmem_symlink_inline_operations;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
2007-10-16 10:25:03 +02:00
|
|
|
* Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin;
|
2007-06-04 10:00:39 +02:00
|
|
|
* but providing them allows a tmpfs file to be used for splice, sendfile, and
|
|
|
|
* below the loop driver, in the generic fashion that many filesystems support.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2007-06-04 10:00:39 +02:00
|
|
|
static int shmem_readpage(struct file *file, struct page *page)
|
|
|
|
{
|
|
|
|
struct inode *inode = page->mapping->host;
|
|
|
|
int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
|
|
|
|
unlock_page(page);
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
static int
|
2007-10-16 10:25:03 +02:00
|
|
|
shmem_write_begin(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned flags,
|
|
|
|
struct page **pagep, void **fsdata)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2007-10-16 10:25:03 +02:00
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
pgoff_t index = pos >> PAGE_CACHE_SHIFT;
|
|
|
|
*pagep = NULL;
|
|
|
|
return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
shmem_write_end(struct file *file, struct address_space *mapping,
|
|
|
|
loff_t pos, unsigned len, unsigned copied,
|
|
|
|
struct page *page, void *fsdata)
|
|
|
|
{
|
|
|
|
struct inode *inode = mapping->host;
|
|
|
|
|
2008-02-05 07:28:44 +01:00
|
|
|
if (pos + copied > inode->i_size)
|
|
|
|
i_size_write(inode, pos + copied);
|
|
|
|
|
|
|
|
unlock_page(page);
|
2007-10-16 10:25:03 +02:00
|
|
|
set_page_dirty(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
|
|
|
|
return copied;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
|
|
|
|
{
|
2006-12-08 11:36:44 +01:00
|
|
|
struct inode *inode = filp->f_path.dentry->d_inode;
|
2005-04-17 00:20:36 +02:00
|
|
|
struct address_space *mapping = inode->i_mapping;
|
|
|
|
unsigned long index, offset;
|
2008-02-05 07:28:51 +01:00
|
|
|
enum sgp_type sgp = SGP_READ;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Might this read be for a stacking filesystem? Then when reading
|
|
|
|
* holes of a sparse file, we actually need to allocate those pages,
|
|
|
|
* and even mark them dirty, so it cannot exceed the max_blocks limit.
|
|
|
|
*/
|
|
|
|
if (segment_eq(get_fs(), KERNEL_DS))
|
|
|
|
sgp = SGP_DIRTY;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
index = *ppos >> PAGE_CACHE_SHIFT;
|
|
|
|
offset = *ppos & ~PAGE_CACHE_MASK;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
struct page *page = NULL;
|
|
|
|
unsigned long end_index, nr, ret;
|
|
|
|
loff_t i_size = i_size_read(inode);
|
|
|
|
|
|
|
|
end_index = i_size >> PAGE_CACHE_SHIFT;
|
|
|
|
if (index > end_index)
|
|
|
|
break;
|
|
|
|
if (index == end_index) {
|
|
|
|
nr = i_size & ~PAGE_CACHE_MASK;
|
|
|
|
if (nr <= offset)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2008-02-05 07:28:51 +01:00
|
|
|
desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (desc->error) {
|
|
|
|
if (desc->error == -EINVAL)
|
|
|
|
desc->error = 0;
|
|
|
|
break;
|
|
|
|
}
|
2008-02-05 07:28:44 +01:00
|
|
|
if (page)
|
|
|
|
unlock_page(page);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We must evaluate after, since reads (unlike writes)
|
2006-01-10 00:59:24 +01:00
|
|
|
* are called without i_mutex protection against truncate
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
nr = PAGE_CACHE_SIZE;
|
|
|
|
i_size = i_size_read(inode);
|
|
|
|
end_index = i_size >> PAGE_CACHE_SHIFT;
|
|
|
|
if (index == end_index) {
|
|
|
|
nr = i_size & ~PAGE_CACHE_MASK;
|
|
|
|
if (nr <= offset) {
|
|
|
|
if (page)
|
|
|
|
page_cache_release(page);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
nr -= offset;
|
|
|
|
|
|
|
|
if (page) {
|
|
|
|
/*
|
|
|
|
* If users can be writing to this page using arbitrary
|
|
|
|
* virtual addresses, take care about potential aliasing
|
|
|
|
* before reading the page on the kernel side.
|
|
|
|
*/
|
|
|
|
if (mapping_writably_mapped(mapping))
|
|
|
|
flush_dcache_page(page);
|
|
|
|
/*
|
|
|
|
* Mark the page accessed if we read the beginning.
|
|
|
|
*/
|
|
|
|
if (!offset)
|
|
|
|
mark_page_accessed(page);
|
2005-10-30 02:16:12 +01:00
|
|
|
} else {
|
2005-04-17 00:20:36 +02:00
|
|
|
page = ZERO_PAGE(0);
|
2005-10-30 02:16:12 +01:00
|
|
|
page_cache_get(page);
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Ok, we have the page, and it's up-to-date, so
|
|
|
|
* now we can copy it to user space...
|
|
|
|
*
|
|
|
|
* The actor routine returns how many bytes were actually used..
|
|
|
|
* NOTE! This may not be the same as how much of a user buffer
|
|
|
|
* we filled up (we may be padding etc), so we can only update
|
|
|
|
* "pos" here (the actor routine has to update the user buffer
|
|
|
|
* pointers and the remaining count).
|
|
|
|
*/
|
|
|
|
ret = actor(desc, page, offset, nr);
|
|
|
|
offset += ret;
|
|
|
|
index += offset >> PAGE_CACHE_SHIFT;
|
|
|
|
offset &= ~PAGE_CACHE_MASK;
|
|
|
|
|
|
|
|
page_cache_release(page);
|
|
|
|
if (ret != nr || !desc->count)
|
|
|
|
break;
|
|
|
|
|
|
|
|
cond_resched();
|
|
|
|
}
|
|
|
|
|
|
|
|
*ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
|
|
|
|
file_accessed(filp);
|
|
|
|
}
|
|
|
|
|
2008-07-24 06:27:35 +02:00
|
|
|
static ssize_t shmem_file_aio_read(struct kiocb *iocb,
|
|
|
|
const struct iovec *iov, unsigned long nr_segs, loff_t pos)
|
|
|
|
{
|
|
|
|
struct file *filp = iocb->ki_filp;
|
|
|
|
ssize_t retval;
|
|
|
|
unsigned long seg;
|
|
|
|
size_t count;
|
|
|
|
loff_t *ppos = &iocb->ki_pos;
|
|
|
|
|
|
|
|
retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
|
|
|
|
if (retval)
|
|
|
|
return retval;
|
|
|
|
|
|
|
|
for (seg = 0; seg < nr_segs; seg++) {
|
|
|
|
read_descriptor_t desc;
|
|
|
|
|
|
|
|
desc.written = 0;
|
|
|
|
desc.arg.buf = iov[seg].iov_base;
|
|
|
|
desc.count = iov[seg].iov_len;
|
|
|
|
if (desc.count == 0)
|
|
|
|
continue;
|
|
|
|
desc.error = 0;
|
|
|
|
do_shmem_file_read(filp, ppos, &desc, file_read_actor);
|
|
|
|
retval += desc.written;
|
|
|
|
if (desc.error) {
|
|
|
|
retval = retval ?: desc.error;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (desc.count > 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return retval;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2006-06-23 11:02:58 +02:00
|
|
|
static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-06-23 11:02:58 +02:00
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
buf->f_type = TMPFS_MAGIC;
|
|
|
|
buf->f_bsize = PAGE_CACHE_SIZE;
|
|
|
|
buf->f_namelen = NAME_MAX;
|
2005-06-22 02:15:04 +02:00
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
if (sbinfo->max_blocks) {
|
2005-04-17 00:20:36 +02:00
|
|
|
buf->f_blocks = sbinfo->max_blocks;
|
|
|
|
buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
|
2005-06-22 02:15:04 +02:00
|
|
|
}
|
|
|
|
if (sbinfo->max_inodes) {
|
2005-04-17 00:20:36 +02:00
|
|
|
buf->f_files = sbinfo->max_inodes;
|
|
|
|
buf->f_ffree = sbinfo->free_inodes;
|
|
|
|
}
|
|
|
|
/* else leave those fields 0 like simple_statfs */
|
2005-06-22 02:15:04 +02:00
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* File creation. Allocate an inode, and we're done..
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
|
|
|
|
{
|
2009-02-24 21:51:52 +01:00
|
|
|
struct inode *inode;
|
2005-04-17 00:20:36 +02:00
|
|
|
int error = -ENOSPC;
|
|
|
|
|
2009-02-24 21:51:52 +01:00
|
|
|
inode = shmem_get_inode(dir->i_sb, mode, dev, VM_NORESERVE);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (inode) {
|
2005-09-09 22:01:43 +02:00
|
|
|
error = security_inode_init_security(inode, dir, NULL, NULL,
|
|
|
|
NULL);
|
|
|
|
if (error) {
|
|
|
|
if (error != -EOPNOTSUPP) {
|
|
|
|
iput(inode);
|
|
|
|
return error;
|
|
|
|
}
|
2006-09-29 11:01:35 +02:00
|
|
|
}
|
|
|
|
error = shmem_acl_init(inode, dir);
|
|
|
|
if (error) {
|
|
|
|
iput(inode);
|
|
|
|
return error;
|
2005-09-09 22:01:43 +02:00
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
if (dir->i_mode & S_ISGID) {
|
|
|
|
inode->i_gid = dir->i_gid;
|
|
|
|
if (S_ISDIR(mode))
|
|
|
|
inode->i_mode |= S_ISGID;
|
|
|
|
}
|
|
|
|
dir->i_size += BOGO_DIRENT_SIZE;
|
|
|
|
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
dget(dentry); /* Extra count - pin the dentry in core */
|
|
|
|
}
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
|
|
|
|
if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
|
|
|
|
return error;
|
2006-10-01 08:29:04 +02:00
|
|
|
inc_nlink(dir);
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
|
|
|
|
struct nameidata *nd)
|
|
|
|
{
|
|
|
|
return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Link a file..
|
|
|
|
*/
|
|
|
|
static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
|
|
|
|
{
|
|
|
|
struct inode *inode = old_dentry->d_inode;
|
2008-02-05 07:28:47 +01:00
|
|
|
int ret;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* No ordinary (disk based) filesystem counts links as inodes;
|
|
|
|
* but each new link needs a new dentry, pinning lowmem, and
|
|
|
|
* tmpfs dentries cannot be pruned until they are unlinked.
|
|
|
|
*/
|
2008-02-05 07:28:47 +01:00
|
|
|
ret = shmem_reserve_inode(inode->i_sb);
|
|
|
|
if (ret)
|
|
|
|
goto out;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
dir->i_size += BOGO_DIRENT_SIZE;
|
|
|
|
inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
2006-10-01 08:29:04 +02:00
|
|
|
inc_nlink(inode);
|
2005-04-17 00:20:36 +02:00
|
|
|
atomic_inc(&inode->i_count); /* New dentry reference */
|
|
|
|
dget(dentry); /* Extra pinning count for the created dentry */
|
|
|
|
d_instantiate(dentry, inode);
|
2008-02-05 07:28:47 +01:00
|
|
|
out:
|
|
|
|
return ret;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_unlink(struct inode *dir, struct dentry *dentry)
|
|
|
|
{
|
|
|
|
struct inode *inode = dentry->d_inode;
|
|
|
|
|
2008-02-05 07:28:47 +01:00
|
|
|
if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
|
|
|
|
shmem_free_inode(inode->i_sb);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
dir->i_size -= BOGO_DIRENT_SIZE;
|
|
|
|
inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
2006-10-01 08:29:03 +02:00
|
|
|
drop_nlink(inode);
|
2005-04-17 00:20:36 +02:00
|
|
|
dput(dentry); /* Undo the count from "create" - this does all the work */
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
|
|
|
|
{
|
|
|
|
if (!simple_empty(dentry))
|
|
|
|
return -ENOTEMPTY;
|
|
|
|
|
2006-10-01 08:29:03 +02:00
|
|
|
drop_nlink(dentry->d_inode);
|
|
|
|
drop_nlink(dir);
|
2005-04-17 00:20:36 +02:00
|
|
|
return shmem_unlink(dir, dentry);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The VFS layer already does all the dentry stuff for rename,
|
|
|
|
* we just have to decrement the usage count for the target if
|
|
|
|
* it exists so that the VFS layer correctly free's it when it
|
|
|
|
* gets overwritten.
|
|
|
|
*/
|
|
|
|
static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
|
|
|
|
{
|
|
|
|
struct inode *inode = old_dentry->d_inode;
|
|
|
|
int they_are_dirs = S_ISDIR(inode->i_mode);
|
|
|
|
|
|
|
|
if (!simple_empty(new_dentry))
|
|
|
|
return -ENOTEMPTY;
|
|
|
|
|
|
|
|
if (new_dentry->d_inode) {
|
|
|
|
(void) shmem_unlink(new_dir, new_dentry);
|
|
|
|
if (they_are_dirs)
|
2006-10-01 08:29:03 +02:00
|
|
|
drop_nlink(old_dir);
|
2005-04-17 00:20:36 +02:00
|
|
|
} else if (they_are_dirs) {
|
2006-10-01 08:29:03 +02:00
|
|
|
drop_nlink(old_dir);
|
2006-10-01 08:29:04 +02:00
|
|
|
inc_nlink(new_dir);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
old_dir->i_size -= BOGO_DIRENT_SIZE;
|
|
|
|
new_dir->i_size += BOGO_DIRENT_SIZE;
|
|
|
|
old_dir->i_ctime = old_dir->i_mtime =
|
|
|
|
new_dir->i_ctime = new_dir->i_mtime =
|
|
|
|
inode->i_ctime = CURRENT_TIME;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
int len;
|
|
|
|
struct inode *inode;
|
|
|
|
struct page *page = NULL;
|
|
|
|
char *kaddr;
|
|
|
|
struct shmem_inode_info *info;
|
|
|
|
|
|
|
|
len = strlen(symname) + 1;
|
|
|
|
if (len > PAGE_CACHE_SIZE)
|
|
|
|
return -ENAMETOOLONG;
|
|
|
|
|
2009-02-24 21:51:52 +01:00
|
|
|
inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!inode)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
2005-09-09 22:01:43 +02:00
|
|
|
error = security_inode_init_security(inode, dir, NULL, NULL,
|
|
|
|
NULL);
|
|
|
|
if (error) {
|
|
|
|
if (error != -EOPNOTSUPP) {
|
|
|
|
iput(inode);
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
error = 0;
|
|
|
|
}
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
info = SHMEM_I(inode);
|
|
|
|
inode->i_size = len-1;
|
|
|
|
if (len <= (char *)inode - (char *)info) {
|
|
|
|
/* do it inline */
|
|
|
|
memcpy(info, symname, len);
|
|
|
|
inode->i_op = &shmem_symlink_inline_operations;
|
|
|
|
} else {
|
|
|
|
error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
|
|
|
|
if (error) {
|
|
|
|
iput(inode);
|
|
|
|
return error;
|
|
|
|
}
|
2008-02-05 07:28:44 +01:00
|
|
|
unlock_page(page);
|
2008-07-29 00:46:19 +02:00
|
|
|
inode->i_mapping->a_ops = &shmem_aops;
|
2005-04-17 00:20:36 +02:00
|
|
|
inode->i_op = &shmem_symlink_inode_operations;
|
|
|
|
kaddr = kmap_atomic(page, KM_USER0);
|
|
|
|
memcpy(kaddr, symname, len);
|
|
|
|
kunmap_atomic(kaddr, KM_USER0);
|
|
|
|
set_page_dirty(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
}
|
|
|
|
if (dir->i_mode & S_ISGID)
|
|
|
|
inode->i_gid = dir->i_gid;
|
|
|
|
dir->i_size += BOGO_DIRENT_SIZE;
|
|
|
|
dir->i_ctime = dir->i_mtime = CURRENT_TIME;
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
dget(dentry);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-08-20 03:02:56 +02:00
|
|
|
static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
|
2005-08-20 03:02:56 +02:00
|
|
|
return NULL;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2005-08-20 03:02:56 +02:00
|
|
|
static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
struct page *page = NULL;
|
|
|
|
int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
|
|
|
|
nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
|
2008-02-05 07:28:44 +01:00
|
|
|
if (page)
|
|
|
|
unlock_page(page);
|
2005-08-20 03:02:56 +02:00
|
|
|
return page;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2005-08-20 03:02:56 +02:00
|
|
|
static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
if (!IS_ERR(nd_get_link(nd))) {
|
2005-08-20 03:02:56 +02:00
|
|
|
struct page *page = cookie;
|
2005-04-17 00:20:36 +02:00
|
|
|
kunmap(page);
|
|
|
|
mark_page_accessed(page);
|
|
|
|
page_cache_release(page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_symlink_inline_operations = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.readlink = generic_readlink,
|
|
|
|
.follow_link = shmem_follow_link_inline,
|
|
|
|
};
|
|
|
|
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_symlink_inode_operations = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.truncate = shmem_truncate,
|
|
|
|
.readlink = generic_readlink,
|
|
|
|
.follow_link = shmem_follow_link,
|
|
|
|
.put_link = shmem_put_link,
|
|
|
|
};
|
|
|
|
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
2008-03-20 01:00:41 +01:00
|
|
|
/*
|
2006-09-29 11:01:35 +02:00
|
|
|
* Superblocks without xattr inode operations will get security.* xattr
|
|
|
|
* support from the VFS "for free". As soon as we have any other xattrs
|
|
|
|
* like ACLs, we also need to implement the security.* handlers at
|
|
|
|
* filesystem level, though.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static size_t shmem_xattr_security_list(struct inode *inode, char *list,
|
|
|
|
size_t list_len, const char *name,
|
|
|
|
size_t name_len)
|
|
|
|
{
|
|
|
|
return security_inode_listsecurity(inode, list, list_len);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_xattr_security_get(struct inode *inode, const char *name,
|
|
|
|
void *buffer, size_t size)
|
|
|
|
{
|
|
|
|
if (strcmp(name, "") == 0)
|
|
|
|
return -EINVAL;
|
2008-02-05 07:29:39 +01:00
|
|
|
return xattr_getsecurity(inode, name, buffer, size);
|
2006-09-29 11:01:35 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_xattr_security_set(struct inode *inode, const char *name,
|
|
|
|
const void *value, size_t size, int flags)
|
|
|
|
{
|
|
|
|
if (strcmp(name, "") == 0)
|
|
|
|
return -EINVAL;
|
|
|
|
return security_inode_setsecurity(inode, name, value, size, flags);
|
|
|
|
}
|
|
|
|
|
2006-12-07 05:38:21 +01:00
|
|
|
static struct xattr_handler shmem_xattr_security_handler = {
|
2006-09-29 11:01:35 +02:00
|
|
|
.prefix = XATTR_SECURITY_PREFIX,
|
|
|
|
.list = shmem_xattr_security_list,
|
|
|
|
.get = shmem_xattr_security_get,
|
|
|
|
.set = shmem_xattr_security_set,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct xattr_handler *shmem_xattr_handlers[] = {
|
|
|
|
&shmem_xattr_acl_access_handler,
|
|
|
|
&shmem_xattr_acl_default_handler,
|
|
|
|
&shmem_xattr_security_handler,
|
|
|
|
NULL
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2006-10-17 09:09:45 +02:00
|
|
|
static struct dentry *shmem_get_parent(struct dentry *child)
|
|
|
|
{
|
|
|
|
return ERR_PTR(-ESTALE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_match(struct inode *ino, void *vfh)
|
|
|
|
{
|
|
|
|
__u32 *fh = vfh;
|
|
|
|
__u64 inum = fh[2];
|
|
|
|
inum = (inum << 32) | fh[1];
|
|
|
|
return ino->i_ino == inum && fh[0] == ino->i_generation;
|
|
|
|
}
|
|
|
|
|
2007-10-22 01:42:13 +02:00
|
|
|
static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
|
|
|
|
struct fid *fid, int fh_len, int fh_type)
|
2006-10-17 09:09:45 +02:00
|
|
|
{
|
|
|
|
struct inode *inode;
|
2007-10-22 01:42:13 +02:00
|
|
|
struct dentry *dentry = NULL;
|
|
|
|
u64 inum = fid->raw[2];
|
|
|
|
inum = (inum << 32) | fid->raw[1];
|
|
|
|
|
|
|
|
if (fh_len < 3)
|
|
|
|
return NULL;
|
2006-10-17 09:09:45 +02:00
|
|
|
|
2007-10-22 01:42:13 +02:00
|
|
|
inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
|
|
|
|
shmem_match, fid->raw);
|
2006-10-17 09:09:45 +02:00
|
|
|
if (inode) {
|
2007-10-22 01:42:13 +02:00
|
|
|
dentry = d_find_alias(inode);
|
2006-10-17 09:09:45 +02:00
|
|
|
iput(inode);
|
|
|
|
}
|
|
|
|
|
2007-10-22 01:42:13 +02:00
|
|
|
return dentry;
|
2006-10-17 09:09:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
|
|
|
|
int connectable)
|
|
|
|
{
|
|
|
|
struct inode *inode = dentry->d_inode;
|
|
|
|
|
|
|
|
if (*len < 3)
|
|
|
|
return 255;
|
|
|
|
|
|
|
|
if (hlist_unhashed(&inode->i_hash)) {
|
|
|
|
/* Unfortunately insert_inode_hash is not idempotent,
|
|
|
|
* so as we hash inodes here rather than at creation
|
|
|
|
* time, we need a lock to ensure we only try
|
|
|
|
* to do it once
|
|
|
|
*/
|
|
|
|
static DEFINE_SPINLOCK(lock);
|
|
|
|
spin_lock(&lock);
|
|
|
|
if (hlist_unhashed(&inode->i_hash))
|
|
|
|
__insert_inode_hash(inode,
|
|
|
|
inode->i_ino + inode->i_generation);
|
|
|
|
spin_unlock(&lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
fh[0] = inode->i_generation;
|
|
|
|
fh[1] = inode->i_ino;
|
|
|
|
fh[2] = ((__u64)inode->i_ino) >> 32;
|
|
|
|
|
|
|
|
*len = 3;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2007-10-22 01:42:17 +02:00
|
|
|
static const struct export_operations shmem_export_ops = {
|
2006-10-17 09:09:45 +02:00
|
|
|
.get_parent = shmem_get_parent,
|
|
|
|
.encode_fh = shmem_encode_fh,
|
2007-10-22 01:42:13 +02:00
|
|
|
.fh_to_dentry = shmem_fh_to_dentry,
|
2006-10-17 09:09:45 +02:00
|
|
|
};
|
|
|
|
|
2008-02-08 13:21:48 +01:00
|
|
|
static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
|
|
|
|
bool remount)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
char *this_char, *value, *rest;
|
|
|
|
|
2006-02-22 00:49:47 +01:00
|
|
|
while (options != NULL) {
|
|
|
|
this_char = options;
|
|
|
|
for (;;) {
|
|
|
|
/*
|
|
|
|
* NUL-terminate this option: unfortunately,
|
|
|
|
* mount options form a comma-separated list,
|
|
|
|
* but mpol's nodelist may also contain commas.
|
|
|
|
*/
|
|
|
|
options = strchr(options, ',');
|
|
|
|
if (options == NULL)
|
|
|
|
break;
|
|
|
|
options++;
|
|
|
|
if (!isdigit(*options)) {
|
|
|
|
options[-1] = '\0';
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!*this_char)
|
|
|
|
continue;
|
|
|
|
if ((value = strchr(this_char,'=')) != NULL) {
|
|
|
|
*value++ = 0;
|
|
|
|
} else {
|
|
|
|
printk(KERN_ERR
|
|
|
|
"tmpfs: No value for mount option '%s'\n",
|
|
|
|
this_char);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!strcmp(this_char,"size")) {
|
|
|
|
unsigned long long size;
|
|
|
|
size = memparse(value,&rest);
|
|
|
|
if (*rest == '%') {
|
|
|
|
size <<= PAGE_SHIFT;
|
|
|
|
size *= totalram_pages;
|
|
|
|
do_div(size, 100);
|
|
|
|
rest++;
|
|
|
|
}
|
|
|
|
if (*rest)
|
|
|
|
goto bad_val;
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->max_blocks =
|
|
|
|
DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
|
2005-04-17 00:20:36 +02:00
|
|
|
} else if (!strcmp(this_char,"nr_blocks")) {
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->max_blocks = memparse(value, &rest);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (*rest)
|
|
|
|
goto bad_val;
|
|
|
|
} else if (!strcmp(this_char,"nr_inodes")) {
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->max_inodes = memparse(value, &rest);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (*rest)
|
|
|
|
goto bad_val;
|
|
|
|
} else if (!strcmp(this_char,"mode")) {
|
2008-02-08 13:21:48 +01:00
|
|
|
if (remount)
|
2005-04-17 00:20:36 +02:00
|
|
|
continue;
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
|
2005-04-17 00:20:36 +02:00
|
|
|
if (*rest)
|
|
|
|
goto bad_val;
|
|
|
|
} else if (!strcmp(this_char,"uid")) {
|
2008-02-08 13:21:48 +01:00
|
|
|
if (remount)
|
2005-04-17 00:20:36 +02:00
|
|
|
continue;
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->uid = simple_strtoul(value, &rest, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (*rest)
|
|
|
|
goto bad_val;
|
|
|
|
} else if (!strcmp(this_char,"gid")) {
|
2008-02-08 13:21:48 +01:00
|
|
|
if (remount)
|
2005-04-17 00:20:36 +02:00
|
|
|
continue;
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->gid = simple_strtoul(value, &rest, 0);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (*rest)
|
|
|
|
goto bad_val;
|
2006-01-14 22:20:48 +01:00
|
|
|
} else if (!strcmp(this_char,"mpol")) {
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
if (mpol_parse_str(value, &sbinfo->mpol, 1))
|
2006-01-14 22:20:48 +01:00
|
|
|
goto bad_val;
|
2005-04-17 00:20:36 +02:00
|
|
|
} else {
|
|
|
|
printk(KERN_ERR "tmpfs: Bad mount option %s\n",
|
|
|
|
this_char);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
bad_val:
|
|
|
|
printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
|
|
|
|
value, this_char);
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
|
|
|
|
{
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
|
2008-02-08 13:21:48 +01:00
|
|
|
struct shmem_sb_info config = *sbinfo;
|
2005-06-22 02:15:04 +02:00
|
|
|
unsigned long blocks;
|
|
|
|
unsigned long inodes;
|
|
|
|
int error = -EINVAL;
|
|
|
|
|
2008-02-08 13:21:48 +01:00
|
|
|
if (shmem_parse_options(data, &config, true))
|
2005-06-22 02:15:04 +02:00
|
|
|
return error;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2005-06-22 02:15:04 +02:00
|
|
|
spin_lock(&sbinfo->stat_lock);
|
|
|
|
blocks = sbinfo->max_blocks - sbinfo->free_blocks;
|
|
|
|
inodes = sbinfo->max_inodes - sbinfo->free_inodes;
|
2008-02-08 13:21:48 +01:00
|
|
|
if (config.max_blocks < blocks)
|
2005-06-22 02:15:04 +02:00
|
|
|
goto out;
|
2008-02-08 13:21:48 +01:00
|
|
|
if (config.max_inodes < inodes)
|
2005-06-22 02:15:04 +02:00
|
|
|
goto out;
|
|
|
|
/*
|
|
|
|
* Those tests also disallow limited->unlimited while any are in
|
|
|
|
* use, so i_blocks will always be zero when max_blocks is zero;
|
|
|
|
* but we must separately disallow unlimited->limited, because
|
|
|
|
* in that case we have no record of how much is already in use.
|
|
|
|
*/
|
2008-02-08 13:21:48 +01:00
|
|
|
if (config.max_blocks && !sbinfo->max_blocks)
|
2005-06-22 02:15:04 +02:00
|
|
|
goto out;
|
2008-02-08 13:21:48 +01:00
|
|
|
if (config.max_inodes && !sbinfo->max_inodes)
|
2005-06-22 02:15:04 +02:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
error = 0;
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->max_blocks = config.max_blocks;
|
|
|
|
sbinfo->free_blocks = config.max_blocks - blocks;
|
|
|
|
sbinfo->max_inodes = config.max_inodes;
|
|
|
|
sbinfo->free_inodes = config.max_inodes - inodes;
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
|
|
|
|
mpol_put(sbinfo->mpol);
|
|
|
|
sbinfo->mpol = config.mpol; /* transfers initial ref */
|
2005-06-22 02:15:04 +02:00
|
|
|
out:
|
|
|
|
spin_unlock(&sbinfo->stat_lock);
|
|
|
|
return error;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2008-02-08 13:21:48 +01:00
|
|
|
|
|
|
|
static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
|
|
|
|
{
|
|
|
|
struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
|
|
|
|
|
|
|
|
if (sbinfo->max_blocks != shmem_default_max_blocks())
|
|
|
|
seq_printf(seq, ",size=%luk",
|
|
|
|
sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
|
|
|
|
if (sbinfo->max_inodes != shmem_default_max_inodes())
|
|
|
|
seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
|
|
|
|
if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
|
|
|
|
seq_printf(seq, ",mode=%03o", sbinfo->mode);
|
|
|
|
if (sbinfo->uid != 0)
|
|
|
|
seq_printf(seq, ",uid=%u", sbinfo->uid);
|
|
|
|
if (sbinfo->gid != 0)
|
|
|
|
seq_printf(seq, ",gid=%u", sbinfo->gid);
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
shmem_show_mpol(seq, sbinfo->mpol);
|
2008-02-08 13:21:48 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_TMPFS */
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
static void shmem_put_super(struct super_block *sb)
|
|
|
|
{
|
|
|
|
kfree(sb->s_fs_info);
|
|
|
|
sb->s_fs_info = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int shmem_fill_super(struct super_block *sb,
|
|
|
|
void *data, int silent)
|
|
|
|
{
|
|
|
|
struct inode *inode;
|
|
|
|
struct dentry *root;
|
2005-06-22 02:15:04 +02:00
|
|
|
struct shmem_sb_info *sbinfo;
|
2008-02-08 13:21:48 +01:00
|
|
|
int err = -ENOMEM;
|
|
|
|
|
|
|
|
/* Round up to L1_CACHE_BYTES to resist false sharing */
|
|
|
|
sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
|
|
|
|
L1_CACHE_BYTES), GFP_KERNEL);
|
|
|
|
if (!sbinfo)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
sbinfo->max_blocks = 0;
|
|
|
|
sbinfo->max_inodes = 0;
|
|
|
|
sbinfo->mode = S_IRWXUGO | S_ISVTX;
|
2008-11-14 00:39:12 +01:00
|
|
|
sbinfo->uid = current_fsuid();
|
|
|
|
sbinfo->gid = current_fsgid();
|
mempolicy: use struct mempolicy pointer in shmem_sb_info
This patch replaces the mempolicy mode, mode_flags, and nodemask in the
shmem_sb_info struct with a struct mempolicy pointer, initialized to NULL.
This removes dependency on the details of mempolicy from shmem.c and hugetlbfs
inode.c and simplifies the interfaces.
mpol_parse_str() in mempolicy.c is changed to return, via a pointer to a
pointer arg, a struct mempolicy pointer on success. For MPOL_DEFAULT, the
returned pointer is NULL. Further, mpol_parse_str() now takes a 'no_context'
argument that causes the input nodemask to be stored in the w.user_nodemask of
the created mempolicy for use when the mempolicy is installed in a tmpfs inode
shared policy tree. At that time, any cpuset contextualization is applied to
the original input nodemask. This preserves the previous behavior where the
input nodemask was stored in the superblock. We can think of the returned
mempolicy as "context free".
Because mpol_parse_str() is now calling mpol_new(), we can remove from
mpol_to_str() the semantic checks that mpol_new() already performs.
Add 'no_context' parameter to mpol_to_str() to specify that it should format
the nodemask in w.user_nodemask for 'bind' and 'interleave' policies.
Change mpol_shared_policy_init() to take a pointer to a "context free" struct
mempolicy and to create a new, "contextualized" mempolicy using the mode,
mode_flags and user_nodemask from the input mempolicy.
Note: we know that the mempolicy passed to mpol_to_str() or
mpol_shared_policy_init() from a tmpfs superblock is "context free". This
is currently the only instance thereof. However, if we found more uses for
this concept, and introduced any ambiguity as to whether a mempolicy was
context free or not, we could add another internal mode flag to identify
context free mempolicies. Then, we could remove the 'no_context' argument
from mpol_to_str().
Added shmem_get_sbmpol() to return a reference counted superblock mempolicy,
if one exists, to pass to mpol_shared_policy_init(). We must add the
reference under the sb stat_lock to prevent races with replacement of the mpol
by remount. This reference is removed in mpol_shared_policy_init().
[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: another build fix]
[akpm@linux-foundation.org: yet another build fix]
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-28 11:13:26 +02:00
|
|
|
sbinfo->mpol = NULL;
|
2008-02-08 13:21:48 +01:00
|
|
|
sb->s_fs_info = sbinfo;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2005-06-22 02:15:04 +02:00
|
|
|
#ifdef CONFIG_TMPFS
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* Per default we only allow half of the physical ram per
|
|
|
|
* tmpfs instance, limiting inodes to one per page of lowmem;
|
|
|
|
* but the internal instance is left unlimited.
|
|
|
|
*/
|
|
|
|
if (!(sb->s_flags & MS_NOUSER)) {
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->max_blocks = shmem_default_max_blocks();
|
|
|
|
sbinfo->max_inodes = shmem_default_max_inodes();
|
|
|
|
if (shmem_parse_options(data, sbinfo, false)) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto failed;
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-10-17 09:09:45 +02:00
|
|
|
sb->s_export_op = &shmem_export_ops;
|
2005-04-17 00:20:36 +02:00
|
|
|
#else
|
|
|
|
sb->s_flags |= MS_NOUSER;
|
|
|
|
#endif
|
|
|
|
|
2005-06-22 02:15:04 +02:00
|
|
|
spin_lock_init(&sbinfo->stat_lock);
|
2008-02-08 13:21:48 +01:00
|
|
|
sbinfo->free_blocks = sbinfo->max_blocks;
|
|
|
|
sbinfo->free_inodes = sbinfo->max_inodes;
|
2005-06-22 02:15:04 +02:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
sb->s_maxbytes = SHMEM_MAX_BYTES;
|
|
|
|
sb->s_blocksize = PAGE_CACHE_SIZE;
|
|
|
|
sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
|
|
|
|
sb->s_magic = TMPFS_MAGIC;
|
|
|
|
sb->s_op = &shmem_ops;
|
[PATCH] tmpfs: time granularity fix for [acm]time going backwards
I noticed a strange behavior in a tmpfs file system the other day, while
building packages - occasionally, and seemingly at random, make decided to
rebuild a target. However, only on tmpfs.
A file would be created, and if checked, it had a sub-second timestamp.
However, after an utimes related call where sub-seconds should be set, they
were zeroed instead. In the case that a file was created, and utimes(...,NULL)
was used on it in the same second, the timestamp on the file moved backwards.
After some digging, I found that this was being caused by tmpfs not having a
time granularity set, thus inheriting the default 1 second granularity.
Hugh adds: yes, we missed tmpfs when the s_time_gran mods went into 2.6.11.
Unfortunately, the granularity of CURRENT_TIME, often used in filesystems,
does not match the default granularity set by alloc_super. A few more such
discrepancies have been found, but this is the most important to fix now.
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
Acked-by: Andi Kleen <ak@suse.de>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-12 22:50:25 +02:00
|
|
|
sb->s_time_gran = 1;
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
|
|
|
sb->s_xattr = shmem_xattr_handlers;
|
|
|
|
sb->s_flags |= MS_POSIXACL;
|
|
|
|
#endif
|
2005-06-22 02:15:04 +02:00
|
|
|
|
2009-02-24 21:51:52 +01:00
|
|
|
inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!inode)
|
|
|
|
goto failed;
|
2008-02-08 13:21:48 +01:00
|
|
|
inode->i_uid = sbinfo->uid;
|
|
|
|
inode->i_gid = sbinfo->gid;
|
2005-04-17 00:20:36 +02:00
|
|
|
root = d_alloc_root(inode);
|
|
|
|
if (!root)
|
|
|
|
goto failed_iput;
|
|
|
|
sb->s_root = root;
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
failed_iput:
|
|
|
|
iput(inode);
|
|
|
|
failed:
|
|
|
|
shmem_put_super(sb);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2006-03-22 09:08:13 +01:00
|
|
|
static struct kmem_cache *shmem_inode_cachep;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
|
|
|
static struct inode *shmem_alloc_inode(struct super_block *sb)
|
|
|
|
{
|
|
|
|
struct shmem_inode_info *p;
|
2006-12-07 05:33:17 +01:00
|
|
|
p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!p)
|
|
|
|
return NULL;
|
|
|
|
return &p->vfs_inode;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void shmem_destroy_inode(struct inode *inode)
|
|
|
|
{
|
|
|
|
if ((inode->i_mode & S_IFMT) == S_IFREG) {
|
|
|
|
/* only struct inode is valid if it's an inline symlink */
|
|
|
|
mpol_free_shared_policy(&SHMEM_I(inode)->policy);
|
|
|
|
}
|
2006-09-29 11:01:35 +02:00
|
|
|
shmem_acl_destroy_inode(inode);
|
2005-04-17 00:20:36 +02:00
|
|
|
kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
|
|
|
|
}
|
|
|
|
|
2008-07-26 04:45:34 +02:00
|
|
|
static void init_once(void *foo)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
|
|
|
|
|
2007-05-17 07:10:57 +02:00
|
|
|
inode_init_once(&p->vfs_inode);
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
2007-05-17 07:10:57 +02:00
|
|
|
p->i_acl = NULL;
|
|
|
|
p->i_default_acl = NULL;
|
2006-09-29 11:01:35 +02:00
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static int init_inodecache(void)
|
|
|
|
{
|
|
|
|
shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
|
|
|
|
sizeof(struct shmem_inode_info),
|
2007-10-17 08:26:10 +02:00
|
|
|
0, SLAB_PANIC, init_once);
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void destroy_inodecache(void)
|
|
|
|
{
|
2006-09-27 10:49:40 +02:00
|
|
|
kmem_cache_destroy(shmem_inode_cachep);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2006-06-28 13:26:44 +02:00
|
|
|
static const struct address_space_operations shmem_aops = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.writepage = shmem_writepage,
|
2007-02-10 10:43:15 +01:00
|
|
|
.set_page_dirty = __set_page_dirty_no_writeback,
|
2005-04-17 00:20:36 +02:00
|
|
|
#ifdef CONFIG_TMPFS
|
2007-06-04 10:00:39 +02:00
|
|
|
.readpage = shmem_readpage,
|
2007-10-16 10:25:03 +02:00
|
|
|
.write_begin = shmem_write_begin,
|
|
|
|
.write_end = shmem_write_end,
|
2005-04-17 00:20:36 +02:00
|
|
|
#endif
|
[PATCH] add migratepage address space op to shmem
Basic problem: pages of a shared memory segment can only be migrated once.
In 2.6.16 through 2.6.17-rc1, shared memory mappings do not have a
migratepage address space op. Therefore, migrate_pages() falls back to
default processing. In this path, it will try to pageout() dirty pages.
Once a shared memory page has been migrated it becomes dirty, so
migrate_pages() will try to page it out. However, because the page count
is 3 [cache + current + pte], pageout() will return PAGE_KEEP because
is_page_cache_freeable() returns false. This will abort all subsequent
migrations.
This patch adds a migratepage address space op to shared memory segments to
avoid taking the default path. We use the "migrate_page()" function
because it knows how to migrate dirty pages. This allows shared memory
segment pages to migrate, subject to other conditions such as # pte's
referencing the page [page_mapcount(page)], when requested.
I think this is safe. If we're migrating a shared memory page, then we
found the page via a page table, so it must be in memory.
Can be verified with memtoy and the shmem-mbind-test script, both
available at: http://free.linux.hp.com/~lts/Tools/
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-04-22 11:35:48 +02:00
|
|
|
.migratepage = migrate_page,
|
2005-04-17 00:20:36 +02:00
|
|
|
};
|
|
|
|
|
2006-12-07 05:40:36 +01:00
|
|
|
static const struct file_operations shmem_file_operations = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.mmap = shmem_mmap,
|
|
|
|
#ifdef CONFIG_TMPFS
|
|
|
|
.llseek = generic_file_llseek,
|
2008-07-24 06:27:35 +02:00
|
|
|
.read = do_sync_read,
|
2008-02-05 07:28:44 +01:00
|
|
|
.write = do_sync_write,
|
2008-07-24 06:27:35 +02:00
|
|
|
.aio_read = shmem_file_aio_read,
|
2008-02-05 07:28:44 +01:00
|
|
|
.aio_write = generic_file_aio_write,
|
2005-04-17 00:20:36 +02:00
|
|
|
.fsync = simple_sync_file,
|
2007-06-04 10:00:39 +02:00
|
|
|
.splice_read = generic_file_splice_read,
|
|
|
|
.splice_write = generic_file_splice_write,
|
2005-04-17 00:20:36 +02:00
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_inode_operations = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.truncate = shmem_truncate,
|
|
|
|
.setattr = shmem_notify_change,
|
[PATCH] madvise(MADV_REMOVE): remove pages from tmpfs shm backing store
Here is the patch to implement madvise(MADV_REMOVE) - which frees up a
given range of pages & its associated backing store. Current
implementation supports only shmfs/tmpfs and other filesystems return
-ENOSYS.
"Some app allocates large tmpfs files, then when some task quits and some
client disconnect, some memory can be released. However the only way to
release tmpfs-swap is to MADV_REMOVE". - Andrea Arcangeli
Databases want to use this feature to drop a section of their bufferpool
(shared memory segments) - without writing back to disk/swap space.
This feature is also useful for supporting hot-plug memory on UML.
Concerns raised by Andrew Morton:
- "We have no plan for holepunching! If we _do_ have such a plan (or
might in the future) then what would the API look like? I think
sys_holepunch(fd, start, len), so we should start out with that."
- Using madvise is very weird, because people will ask "why do I need to
mmap my file before I can stick a hole in it?"
- None of the other madvise operations call into the filesystem in this
manner. A broad question is: is this capability an MM operation or a
filesytem operation? truncate, for example, is a filesystem operation
which sometimes has MM side-effects. madvise is an mm operation and with
this patch, it gains FS side-effects, only they're really, really
significant ones."
Comments:
- Andrea suggested the fs operation too but then it's more efficient to
have it as a mm operation with fs side effects, because they don't
immediatly know fd and physical offset of the range. It's possible to
fixup in userland and to use the fs operation but it's more expensive,
the vmas are already in the kernel and we can use them.
Short term plan & Future Direction:
- We seem to need this interface only for shmfs/tmpfs files in the short
term. We have to add hooks into the filesystem for correctness and
completeness. This is what this patch does.
- In the future, plan is to support both fs and mmap apis also. This
also involves (other) filesystem specific functions to be implemented.
- Current patch doesn't support VM_NONLINEAR - which can be addressed in
the future.
Signed-off-by: Badari Pulavarty <pbadari@us.ibm.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Andrea Arcangeli <andrea@suse.de>
Cc: Michael Kerrisk <mtk-manpages@gmx.net>
Cc: Ulrich Drepper <drepper@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-01-06 09:10:38 +01:00
|
|
|
.truncate_range = shmem_truncate_range,
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
|
|
|
.setxattr = generic_setxattr,
|
|
|
|
.getxattr = generic_getxattr,
|
|
|
|
.listxattr = generic_listxattr,
|
|
|
|
.removexattr = generic_removexattr,
|
|
|
|
.permission = shmem_permission,
|
|
|
|
#endif
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
};
|
|
|
|
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_dir_inode_operations = {
|
2005-04-17 00:20:36 +02:00
|
|
|
#ifdef CONFIG_TMPFS
|
|
|
|
.create = shmem_create,
|
|
|
|
.lookup = simple_lookup,
|
|
|
|
.link = shmem_link,
|
|
|
|
.unlink = shmem_unlink,
|
|
|
|
.symlink = shmem_symlink,
|
|
|
|
.mkdir = shmem_mkdir,
|
|
|
|
.rmdir = shmem_rmdir,
|
|
|
|
.mknod = shmem_mknod,
|
|
|
|
.rename = shmem_rename,
|
|
|
|
#endif
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
|
|
|
.setattr = shmem_notify_change,
|
|
|
|
.setxattr = generic_setxattr,
|
|
|
|
.getxattr = generic_getxattr,
|
|
|
|
.listxattr = generic_listxattr,
|
|
|
|
.removexattr = generic_removexattr,
|
|
|
|
.permission = shmem_permission,
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
2007-02-12 09:55:39 +01:00
|
|
|
static const struct inode_operations shmem_special_inode_operations = {
|
2006-09-29 11:01:35 +02:00
|
|
|
#ifdef CONFIG_TMPFS_POSIX_ACL
|
|
|
|
.setattr = shmem_notify_change,
|
|
|
|
.setxattr = generic_setxattr,
|
|
|
|
.getxattr = generic_getxattr,
|
|
|
|
.listxattr = generic_listxattr,
|
|
|
|
.removexattr = generic_removexattr,
|
|
|
|
.permission = shmem_permission,
|
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
};
|
|
|
|
|
2007-03-05 09:30:28 +01:00
|
|
|
static const struct super_operations shmem_ops = {
|
2005-04-17 00:20:36 +02:00
|
|
|
.alloc_inode = shmem_alloc_inode,
|
|
|
|
.destroy_inode = shmem_destroy_inode,
|
|
|
|
#ifdef CONFIG_TMPFS
|
|
|
|
.statfs = shmem_statfs,
|
|
|
|
.remount_fs = shmem_remount_fs,
|
2008-02-08 13:21:48 +01:00
|
|
|
.show_options = shmem_show_options,
|
2005-04-17 00:20:36 +02:00
|
|
|
#endif
|
|
|
|
.delete_inode = shmem_delete_inode,
|
|
|
|
.drop_inode = generic_delete_inode,
|
|
|
|
.put_super = shmem_put_super,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct vm_operations_struct shmem_vm_ops = {
|
2007-07-19 10:46:59 +02:00
|
|
|
.fault = shmem_fault,
|
2005-04-17 00:20:36 +02:00
|
|
|
#ifdef CONFIG_NUMA
|
|
|
|
.set_policy = shmem_set_policy,
|
|
|
|
.get_policy = shmem_get_policy,
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
|
[PATCH] VFS: Permit filesystem to override root dentry on mount
Extend the get_sb() filesystem operation to take an extra argument that
permits the VFS to pass in the target vfsmount that defines the mountpoint.
The filesystem is then required to manually set the superblock and root dentry
pointers. For most filesystems, this should be done with simple_set_mnt()
which will set the superblock pointer and then set the root dentry to the
superblock's s_root (as per the old default behaviour).
The get_sb() op now returns an integer as there's now no need to return the
superblock pointer.
This patch permits a superblock to be implicitly shared amongst several mount
points, such as can be done with NFS to avoid potential inode aliasing. In
such a case, simple_set_mnt() would not be called, and instead the mnt_root
and mnt_sb would be set directly.
The patch also makes the following changes:
(*) the get_sb_*() convenience functions in the core kernel now take a vfsmount
pointer argument and return an integer, so most filesystems have to change
very little.
(*) If one of the convenience function is not used, then get_sb() should
normally call simple_set_mnt() to instantiate the vfsmount. This will
always return 0, and so can be tail-called from get_sb().
(*) generic_shutdown_super() now calls shrink_dcache_sb() to clean up the
dcache upon superblock destruction rather than shrink_dcache_anon().
This is required because the superblock may now have multiple trees that
aren't actually bound to s_root, but that still need to be cleaned up. The
currently called functions assume that the whole tree is rooted at s_root,
and that anonymous dentries are not the roots of trees which results in
dentries being left unculled.
However, with the way NFS superblock sharing are currently set to be
implemented, these assumptions are violated: the root of the filesystem is
simply a dummy dentry and inode (the real inode for '/' may well be
inaccessible), and all the vfsmounts are rooted on anonymous[*] dentries
with child trees.
[*] Anonymous until discovered from another tree.
(*) The documentation has been adjusted, including the additional bit of
changing ext2_* into foo_* in the documentation.
[akpm@osdl.org: convert ipath_fs, do other stuff]
Signed-off-by: David Howells <dhowells@redhat.com>
Acked-by: Al Viro <viro@zeniv.linux.org.uk>
Cc: Nathan Scott <nathans@sgi.com>
Cc: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 11:02:57 +02:00
|
|
|
static int shmem_get_sb(struct file_system_type *fs_type,
|
|
|
|
int flags, const char *dev_name, void *data, struct vfsmount *mnt)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
[PATCH] VFS: Permit filesystem to override root dentry on mount
Extend the get_sb() filesystem operation to take an extra argument that
permits the VFS to pass in the target vfsmount that defines the mountpoint.
The filesystem is then required to manually set the superblock and root dentry
pointers. For most filesystems, this should be done with simple_set_mnt()
which will set the superblock pointer and then set the root dentry to the
superblock's s_root (as per the old default behaviour).
The get_sb() op now returns an integer as there's now no need to return the
superblock pointer.
This patch permits a superblock to be implicitly shared amongst several mount
points, such as can be done with NFS to avoid potential inode aliasing. In
such a case, simple_set_mnt() would not be called, and instead the mnt_root
and mnt_sb would be set directly.
The patch also makes the following changes:
(*) the get_sb_*() convenience functions in the core kernel now take a vfsmount
pointer argument and return an integer, so most filesystems have to change
very little.
(*) If one of the convenience function is not used, then get_sb() should
normally call simple_set_mnt() to instantiate the vfsmount. This will
always return 0, and so can be tail-called from get_sb().
(*) generic_shutdown_super() now calls shrink_dcache_sb() to clean up the
dcache upon superblock destruction rather than shrink_dcache_anon().
This is required because the superblock may now have multiple trees that
aren't actually bound to s_root, but that still need to be cleaned up. The
currently called functions assume that the whole tree is rooted at s_root,
and that anonymous dentries are not the roots of trees which results in
dentries being left unculled.
However, with the way NFS superblock sharing are currently set to be
implemented, these assumptions are violated: the root of the filesystem is
simply a dummy dentry and inode (the real inode for '/' may well be
inaccessible), and all the vfsmounts are rooted on anonymous[*] dentries
with child trees.
[*] Anonymous until discovered from another tree.
(*) The documentation has been adjusted, including the additional bit of
changing ext2_* into foo_* in the documentation.
[akpm@osdl.org: convert ipath_fs, do other stuff]
Signed-off-by: David Howells <dhowells@redhat.com>
Acked-by: Al Viro <viro@zeniv.linux.org.uk>
Cc: Nathan Scott <nathans@sgi.com>
Cc: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-23 11:02:57 +02:00
|
|
|
return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct file_system_type tmpfs_fs_type = {
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.name = "tmpfs",
|
|
|
|
.get_sb = shmem_get_sb,
|
|
|
|
.kill_sb = kill_litter_super,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int __init init_tmpfs(void)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
|
2007-10-17 08:25:46 +02:00
|
|
|
error = bdi_init(&shmem_backing_dev_info);
|
|
|
|
if (error)
|
|
|
|
goto out4;
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
error = init_inodecache();
|
|
|
|
if (error)
|
|
|
|
goto out3;
|
|
|
|
|
|
|
|
error = register_filesystem(&tmpfs_fs_type);
|
|
|
|
if (error) {
|
|
|
|
printk(KERN_ERR "Could not register tmpfs\n");
|
|
|
|
goto out2;
|
|
|
|
}
|
2005-06-21 06:15:16 +02:00
|
|
|
|
2006-06-09 15:34:16 +02:00
|
|
|
shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
|
2005-04-17 00:20:36 +02:00
|
|
|
tmpfs_fs_type.name, NULL);
|
|
|
|
if (IS_ERR(shm_mnt)) {
|
|
|
|
error = PTR_ERR(shm_mnt);
|
|
|
|
printk(KERN_ERR "Could not kern_mount tmpfs\n");
|
|
|
|
goto out1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
out1:
|
|
|
|
unregister_filesystem(&tmpfs_fs_type);
|
|
|
|
out2:
|
|
|
|
destroy_inodecache();
|
|
|
|
out3:
|
2007-10-17 08:25:46 +02:00
|
|
|
bdi_destroy(&shmem_backing_dev_info);
|
|
|
|
out4:
|
2005-04-17 00:20:36 +02:00
|
|
|
shm_mnt = ERR_PTR(error);
|
|
|
|
return error;
|
|
|
|
}
|
2009-01-06 23:40:20 +01:00
|
|
|
|
|
|
|
#else /* !CONFIG_SHMEM */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* tiny-shmem: simple shmemfs and tmpfs using ramfs code
|
|
|
|
*
|
|
|
|
* This is intended for small system where the benefits of the full
|
|
|
|
* shmem code (swap-backed and resource-limited) are outweighed by
|
|
|
|
* their complexity. On systems without swap this code should be
|
|
|
|
* effectively equivalent, but much lighter weight.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/ramfs.h>
|
|
|
|
|
|
|
|
static struct file_system_type tmpfs_fs_type = {
|
|
|
|
.name = "tmpfs",
|
|
|
|
.get_sb = ramfs_get_sb,
|
|
|
|
.kill_sb = kill_litter_super,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int __init init_tmpfs(void)
|
|
|
|
{
|
|
|
|
BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
|
|
|
|
|
|
|
|
shm_mnt = kern_mount(&tmpfs_fs_type);
|
|
|
|
BUG_ON(IS_ERR(shm_mnt));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int shmem_unuse(swp_entry_t entry, struct page *page)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-02-24 21:51:52 +01:00
|
|
|
#define shmem_vm_ops generic_file_vm_ops
|
|
|
|
#define shmem_file_operations ramfs_file_operations
|
|
|
|
#define shmem_get_inode(sb, mode, dev, flags) ramfs_get_inode(sb, mode, dev)
|
|
|
|
#define shmem_acct_size(flags, size) 0
|
|
|
|
#define shmem_unacct_size(flags, size) do {} while (0)
|
2009-04-13 23:40:12 +02:00
|
|
|
#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE
|
2009-01-06 23:40:20 +01:00
|
|
|
|
|
|
|
#endif /* CONFIG_SHMEM */
|
|
|
|
|
|
|
|
/* common code */
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-03-20 01:00:41 +01:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* shmem_file_setup - get an unlinked file living in tmpfs
|
|
|
|
* @name: name for dentry (to be seen in /proc/<pid>/maps
|
|
|
|
* @size: size to be set for the file
|
2009-02-24 21:51:52 +01:00
|
|
|
* @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
struct file *file;
|
|
|
|
struct inode *inode;
|
|
|
|
struct dentry *dentry, *root;
|
|
|
|
struct qstr this;
|
|
|
|
|
|
|
|
if (IS_ERR(shm_mnt))
|
|
|
|
return (void *)shm_mnt;
|
|
|
|
|
|
|
|
if (size < 0 || size > SHMEM_MAX_BYTES)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
if (shmem_acct_size(flags, size))
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
|
|
error = -ENOMEM;
|
|
|
|
this.name = name;
|
|
|
|
this.len = strlen(name);
|
|
|
|
this.hash = 0; /* will go */
|
|
|
|
root = shm_mnt->mnt_root;
|
|
|
|
dentry = d_alloc(root, &this);
|
|
|
|
if (!dentry)
|
|
|
|
goto put_memory;
|
|
|
|
|
|
|
|
error = -ENFILE;
|
|
|
|
file = get_empty_filp();
|
|
|
|
if (!file)
|
|
|
|
goto put_dentry;
|
|
|
|
|
|
|
|
error = -ENOSPC;
|
2009-02-24 21:51:52 +01:00
|
|
|
inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0, flags);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (!inode)
|
|
|
|
goto close_file;
|
|
|
|
|
|
|
|
d_instantiate(dentry, inode);
|
|
|
|
inode->i_size = size;
|
|
|
|
inode->i_nlink = 0; /* It is unlinked */
|
2007-10-17 08:31:13 +02:00
|
|
|
init_file(file, shm_mnt, dentry, FMODE_WRITE | FMODE_READ,
|
2009-01-06 23:40:20 +01:00
|
|
|
&shmem_file_operations);
|
|
|
|
|
|
|
|
#ifndef CONFIG_MMU
|
|
|
|
error = ramfs_nommu_expand_for_mapping(inode, size);
|
|
|
|
if (error)
|
|
|
|
goto close_file;
|
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
return file;
|
|
|
|
|
|
|
|
close_file:
|
|
|
|
put_filp(file);
|
|
|
|
put_dentry:
|
|
|
|
dput(dentry);
|
|
|
|
put_memory:
|
|
|
|
shmem_unacct_size(flags, size);
|
|
|
|
return ERR_PTR(error);
|
|
|
|
}
|
2008-06-20 09:08:06 +02:00
|
|
|
EXPORT_SYMBOL_GPL(shmem_file_setup);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-03-20 01:00:41 +01:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* shmem_zero_setup - setup a shared anonymous mapping
|
|
|
|
* @vma: the vma to be mmapped is prepared by do_mmap_pgoff
|
|
|
|
*/
|
|
|
|
int shmem_zero_setup(struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct file *file;
|
|
|
|
loff_t size = vma->vm_end - vma->vm_start;
|
|
|
|
|
|
|
|
file = shmem_file_setup("dev/zero", size, vma->vm_flags);
|
|
|
|
if (IS_ERR(file))
|
|
|
|
return PTR_ERR(file);
|
|
|
|
|
2009-05-19 19:25:57 +02:00
|
|
|
ima_counts_get(file);
|
2005-04-17 00:20:36 +02:00
|
|
|
if (vma->vm_file)
|
|
|
|
fput(vma->vm_file);
|
|
|
|
vma->vm_file = file;
|
|
|
|
vma->vm_ops = &shmem_vm_ops;
|
|
|
|
return 0;
|
|
|
|
}
|
2009-01-06 23:40:20 +01:00
|
|
|
|
|
|
|
module_init(init_tmpfs)
|