zsmalloc: collapse internal .h into .c
The patch collapses in the internal zsmalloc_int.h into the zsmalloc-main.c file. This is done in preparation for the promotion to mm/ where separate internal headers are discouraged. Signed-off-by: Seth Jennings <sjenning@linux.vnet.ibm.com> Signed-off-by: Minchan Kim <minchan@kernel.org> Acked-by: Nitin Gupta <ngupta@vflare.org> Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
parent
4ec2601f6f
commit
0959c63f11
|
@ -76,9 +76,139 @@
|
||||||
#include <linux/cpu.h>
|
#include <linux/cpu.h>
|
||||||
#include <linux/vmalloc.h>
|
#include <linux/vmalloc.h>
|
||||||
#include <linux/hardirq.h>
|
#include <linux/hardirq.h>
|
||||||
|
#include <linux/spinlock.h>
|
||||||
|
#include <linux/types.h>
|
||||||
|
|
||||||
#include "zsmalloc.h"
|
#include "zsmalloc.h"
|
||||||
#include "zsmalloc_int.h"
|
|
||||||
|
/*
|
||||||
|
* This must be power of 2 and greater than of equal to sizeof(link_free).
|
||||||
|
* These two conditions ensure that any 'struct link_free' itself doesn't
|
||||||
|
* span more than 1 page which avoids complex case of mapping 2 pages simply
|
||||||
|
* to restore link_free pointer values.
|
||||||
|
*/
|
||||||
|
#define ZS_ALIGN 8
|
||||||
|
|
||||||
|
/*
|
||||||
|
* A single 'zspage' is composed of up to 2^N discontiguous 0-order (single)
|
||||||
|
* pages. ZS_MAX_ZSPAGE_ORDER defines upper limit on N.
|
||||||
|
*/
|
||||||
|
#define ZS_MAX_ZSPAGE_ORDER 2
|
||||||
|
#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Object location (<PFN>, <obj_idx>) is encoded as
|
||||||
|
* as single (void *) handle value.
|
||||||
|
*
|
||||||
|
* Note that object index <obj_idx> is relative to system
|
||||||
|
* page <PFN> it is stored in, so for each sub-page belonging
|
||||||
|
* to a zspage, obj_idx starts with 0.
|
||||||
|
*
|
||||||
|
* This is made more complicated by various memory models and PAE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef MAX_PHYSMEM_BITS
|
||||||
|
#ifdef CONFIG_HIGHMEM64G
|
||||||
|
#define MAX_PHYSMEM_BITS 36
|
||||||
|
#else /* !CONFIG_HIGHMEM64G */
|
||||||
|
/*
|
||||||
|
* If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
|
||||||
|
* be PAGE_SHIFT
|
||||||
|
*/
|
||||||
|
#define MAX_PHYSMEM_BITS BITS_PER_LONG
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
|
||||||
|
#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS)
|
||||||
|
#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
|
||||||
|
|
||||||
|
#define MAX(a, b) ((a) >= (b) ? (a) : (b))
|
||||||
|
/* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
|
||||||
|
#define ZS_MIN_ALLOC_SIZE \
|
||||||
|
MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
|
||||||
|
#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
|
||||||
|
|
||||||
|
/*
|
||||||
|
* On systems with 4K page size, this gives 254 size classes! There is a
|
||||||
|
* trader-off here:
|
||||||
|
* - Large number of size classes is potentially wasteful as free page are
|
||||||
|
* spread across these classes
|
||||||
|
* - Small number of size classes causes large internal fragmentation
|
||||||
|
* - Probably its better to use specific size classes (empirically
|
||||||
|
* determined). NOTE: all those class sizes must be set as multiple of
|
||||||
|
* ZS_ALIGN to make sure link_free itself never has to span 2 pages.
|
||||||
|
*
|
||||||
|
* ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
|
||||||
|
* (reason above)
|
||||||
|
*/
|
||||||
|
#define ZS_SIZE_CLASS_DELTA 16
|
||||||
|
#define ZS_SIZE_CLASSES ((ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) / \
|
||||||
|
ZS_SIZE_CLASS_DELTA + 1)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We do not maintain any list for completely empty or full pages
|
||||||
|
*/
|
||||||
|
enum fullness_group {
|
||||||
|
ZS_ALMOST_FULL,
|
||||||
|
ZS_ALMOST_EMPTY,
|
||||||
|
_ZS_NR_FULLNESS_GROUPS,
|
||||||
|
|
||||||
|
ZS_EMPTY,
|
||||||
|
ZS_FULL
|
||||||
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We assign a page to ZS_ALMOST_EMPTY fullness group when:
|
||||||
|
* n <= N / f, where
|
||||||
|
* n = number of allocated objects
|
||||||
|
* N = total number of objects zspage can store
|
||||||
|
* f = 1/fullness_threshold_frac
|
||||||
|
*
|
||||||
|
* Similarly, we assign zspage to:
|
||||||
|
* ZS_ALMOST_FULL when n > N / f
|
||||||
|
* ZS_EMPTY when n == 0
|
||||||
|
* ZS_FULL when n == N
|
||||||
|
*
|
||||||
|
* (see: fix_fullness_group())
|
||||||
|
*/
|
||||||
|
static const int fullness_threshold_frac = 4;
|
||||||
|
|
||||||
|
struct size_class {
|
||||||
|
/*
|
||||||
|
* Size of objects stored in this class. Must be multiple
|
||||||
|
* of ZS_ALIGN.
|
||||||
|
*/
|
||||||
|
int size;
|
||||||
|
unsigned int index;
|
||||||
|
|
||||||
|
/* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */
|
||||||
|
int pages_per_zspage;
|
||||||
|
|
||||||
|
spinlock_t lock;
|
||||||
|
|
||||||
|
/* stats */
|
||||||
|
u64 pages_allocated;
|
||||||
|
|
||||||
|
struct page *fullness_list[_ZS_NR_FULLNESS_GROUPS];
|
||||||
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Placed within free objects to form a singly linked list.
|
||||||
|
* For every zspage, first_page->freelist gives head of this list.
|
||||||
|
*
|
||||||
|
* This must be power of 2 and less than or equal to ZS_ALIGN
|
||||||
|
*/
|
||||||
|
struct link_free {
|
||||||
|
/* Handle of next free chunk (encodes <PFN, obj_idx>) */
|
||||||
|
void *next;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct zs_pool {
|
||||||
|
struct size_class size_class[ZS_SIZE_CLASSES];
|
||||||
|
|
||||||
|
gfp_t flags; /* allocation flags used when growing pool */
|
||||||
|
const char *name;
|
||||||
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* A zspage's class index and fullness group
|
* A zspage's class index and fullness group
|
||||||
|
|
|
@ -1,149 +0,0 @@
|
||||||
/*
|
|
||||||
* zsmalloc memory allocator
|
|
||||||
*
|
|
||||||
* Copyright (C) 2011 Nitin Gupta
|
|
||||||
*
|
|
||||||
* This code is released using a dual license strategy: BSD/GPL
|
|
||||||
* You can choose the license that better fits your requirements.
|
|
||||||
*
|
|
||||||
* Released under the terms of 3-clause BSD License
|
|
||||||
* Released under the terms of GNU General Public License Version 2.0
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef _ZS_MALLOC_INT_H_
|
|
||||||
#define _ZS_MALLOC_INT_H_
|
|
||||||
|
|
||||||
#include <linux/kernel.h>
|
|
||||||
#include <linux/spinlock.h>
|
|
||||||
#include <linux/types.h>
|
|
||||||
|
|
||||||
/*
|
|
||||||
* This must be power of 2 and greater than of equal to sizeof(link_free).
|
|
||||||
* These two conditions ensure that any 'struct link_free' itself doesn't
|
|
||||||
* span more than 1 page which avoids complex case of mapping 2 pages simply
|
|
||||||
* to restore link_free pointer values.
|
|
||||||
*/
|
|
||||||
#define ZS_ALIGN 8
|
|
||||||
|
|
||||||
/*
|
|
||||||
* A single 'zspage' is composed of up to 2^N discontiguous 0-order (single)
|
|
||||||
* pages. ZS_MAX_ZSPAGE_ORDER defines upper limit on N.
|
|
||||||
*/
|
|
||||||
#define ZS_MAX_ZSPAGE_ORDER 2
|
|
||||||
#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Object location (<PFN>, <obj_idx>) is encoded as
|
|
||||||
* as single (void *) handle value.
|
|
||||||
*
|
|
||||||
* Note that object index <obj_idx> is relative to system
|
|
||||||
* page <PFN> it is stored in, so for each sub-page belonging
|
|
||||||
* to a zspage, obj_idx starts with 0.
|
|
||||||
*
|
|
||||||
* This is made more complicated by various memory models and PAE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef MAX_PHYSMEM_BITS
|
|
||||||
#ifdef CONFIG_HIGHMEM64G
|
|
||||||
#define MAX_PHYSMEM_BITS 36
|
|
||||||
#else /* !CONFIG_HIGHMEM64G */
|
|
||||||
/*
|
|
||||||
* If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
|
|
||||||
* be PAGE_SHIFT
|
|
||||||
*/
|
|
||||||
#define MAX_PHYSMEM_BITS BITS_PER_LONG
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
|
|
||||||
#define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS)
|
|
||||||
#define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
|
|
||||||
|
|
||||||
#define MAX(a, b) ((a) >= (b) ? (a) : (b))
|
|
||||||
/* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
|
|
||||||
#define ZS_MIN_ALLOC_SIZE \
|
|
||||||
MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
|
|
||||||
#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
|
|
||||||
|
|
||||||
/*
|
|
||||||
* On systems with 4K page size, this gives 254 size classes! There is a
|
|
||||||
* trader-off here:
|
|
||||||
* - Large number of size classes is potentially wasteful as free page are
|
|
||||||
* spread across these classes
|
|
||||||
* - Small number of size classes causes large internal fragmentation
|
|
||||||
* - Probably its better to use specific size classes (empirically
|
|
||||||
* determined). NOTE: all those class sizes must be set as multiple of
|
|
||||||
* ZS_ALIGN to make sure link_free itself never has to span 2 pages.
|
|
||||||
*
|
|
||||||
* ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
|
|
||||||
* (reason above)
|
|
||||||
*/
|
|
||||||
#define ZS_SIZE_CLASS_DELTA 16
|
|
||||||
#define ZS_SIZE_CLASSES ((ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) / \
|
|
||||||
ZS_SIZE_CLASS_DELTA + 1)
|
|
||||||
|
|
||||||
/*
|
|
||||||
* We do not maintain any list for completely empty or full pages
|
|
||||||
*/
|
|
||||||
enum fullness_group {
|
|
||||||
ZS_ALMOST_FULL,
|
|
||||||
ZS_ALMOST_EMPTY,
|
|
||||||
_ZS_NR_FULLNESS_GROUPS,
|
|
||||||
|
|
||||||
ZS_EMPTY,
|
|
||||||
ZS_FULL
|
|
||||||
};
|
|
||||||
|
|
||||||
/*
|
|
||||||
* We assign a page to ZS_ALMOST_EMPTY fullness group when:
|
|
||||||
* n <= N / f, where
|
|
||||||
* n = number of allocated objects
|
|
||||||
* N = total number of objects zspage can store
|
|
||||||
* f = 1/fullness_threshold_frac
|
|
||||||
*
|
|
||||||
* Similarly, we assign zspage to:
|
|
||||||
* ZS_ALMOST_FULL when n > N / f
|
|
||||||
* ZS_EMPTY when n == 0
|
|
||||||
* ZS_FULL when n == N
|
|
||||||
*
|
|
||||||
* (see: fix_fullness_group())
|
|
||||||
*/
|
|
||||||
static const int fullness_threshold_frac = 4;
|
|
||||||
|
|
||||||
struct size_class {
|
|
||||||
/*
|
|
||||||
* Size of objects stored in this class. Must be multiple
|
|
||||||
* of ZS_ALIGN.
|
|
||||||
*/
|
|
||||||
int size;
|
|
||||||
unsigned int index;
|
|
||||||
|
|
||||||
/* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */
|
|
||||||
int pages_per_zspage;
|
|
||||||
|
|
||||||
spinlock_t lock;
|
|
||||||
|
|
||||||
/* stats */
|
|
||||||
u64 pages_allocated;
|
|
||||||
|
|
||||||
struct page *fullness_list[_ZS_NR_FULLNESS_GROUPS];
|
|
||||||
};
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Placed within free objects to form a singly linked list.
|
|
||||||
* For every zspage, first_page->freelist gives head of this list.
|
|
||||||
*
|
|
||||||
* This must be power of 2 and less than or equal to ZS_ALIGN
|
|
||||||
*/
|
|
||||||
struct link_free {
|
|
||||||
/* Handle of next free chunk (encodes <PFN, obj_idx>) */
|
|
||||||
void *next;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct zs_pool {
|
|
||||||
struct size_class size_class[ZS_SIZE_CLASSES];
|
|
||||||
|
|
||||||
gfp_t flags; /* allocation flags used when growing pool */
|
|
||||||
const char *name;
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif
|
|
Loading…
Reference in New Issue