percpu: move chunk area map extension out of area allocation
Impact: code reorganization for later changes Separate out chunk area map extension into a separate function - pcpu_extend_area_map() - and call it directly from pcpu_alloc() such that pcpu_alloc_area() is guaranteed to have enough area map slots on invocation. With this change, pcpu_alloc_area() does only area allocation and the only failure mode is when the chunk doens't have enough room, so there's no need to distinguish it from memory allocation failures. Make it return -1 on such cases instead of hacky -ENOSPC. Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
parent
1880d93b80
commit
9f7dcf224b
108
mm/percpu.c
108
mm/percpu.c
|
@ -306,6 +306,50 @@ static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
|
||||||
rb_insert_color(&new->rb_node, &pcpu_addr_root);
|
rb_insert_color(&new->rb_node, &pcpu_addr_root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* pcpu_extend_area_map - extend area map for allocation
|
||||||
|
* @chunk: target chunk
|
||||||
|
*
|
||||||
|
* Extend area map of @chunk so that it can accomodate an allocation.
|
||||||
|
* A single allocation can split an area into three areas, so this
|
||||||
|
* function makes sure that @chunk->map has at least two extra slots.
|
||||||
|
*
|
||||||
|
* RETURNS:
|
||||||
|
* 0 if noop, 1 if successfully extended, -errno on failure.
|
||||||
|
*/
|
||||||
|
static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
|
||||||
|
{
|
||||||
|
int new_alloc;
|
||||||
|
int *new;
|
||||||
|
size_t size;
|
||||||
|
|
||||||
|
/* has enough? */
|
||||||
|
if (chunk->map_alloc >= chunk->map_used + 2)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
new_alloc = PCPU_DFL_MAP_ALLOC;
|
||||||
|
while (new_alloc < chunk->map_used + 2)
|
||||||
|
new_alloc *= 2;
|
||||||
|
|
||||||
|
new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
|
||||||
|
if (!new)
|
||||||
|
return -ENOMEM;
|
||||||
|
|
||||||
|
size = chunk->map_alloc * sizeof(chunk->map[0]);
|
||||||
|
memcpy(new, chunk->map, size);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* map_alloc < PCPU_DFL_MAP_ALLOC indicates that the chunk is
|
||||||
|
* one of the first chunks and still using static map.
|
||||||
|
*/
|
||||||
|
if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC)
|
||||||
|
pcpu_mem_free(chunk->map, size);
|
||||||
|
|
||||||
|
chunk->map_alloc = new_alloc;
|
||||||
|
chunk->map = new;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* pcpu_split_block - split a map block
|
* pcpu_split_block - split a map block
|
||||||
* @chunk: chunk of interest
|
* @chunk: chunk of interest
|
||||||
|
@ -321,44 +365,16 @@ static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
|
||||||
* depending on @head, is reduced by @tail bytes and @tail byte block
|
* depending on @head, is reduced by @tail bytes and @tail byte block
|
||||||
* is inserted after the target block.
|
* is inserted after the target block.
|
||||||
*
|
*
|
||||||
* RETURNS:
|
* @chunk->map must have enough free slots to accomodate the split.
|
||||||
* 0 on success, -errno on failure.
|
|
||||||
*/
|
*/
|
||||||
static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
|
static void pcpu_split_block(struct pcpu_chunk *chunk, int i,
|
||||||
|
int head, int tail)
|
||||||
{
|
{
|
||||||
int nr_extra = !!head + !!tail;
|
int nr_extra = !!head + !!tail;
|
||||||
int target = chunk->map_used + nr_extra;
|
|
||||||
|
|
||||||
/* reallocation required? */
|
BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra);
|
||||||
if (chunk->map_alloc < target) {
|
|
||||||
int new_alloc;
|
|
||||||
int *new;
|
|
||||||
size_t size;
|
|
||||||
|
|
||||||
new_alloc = PCPU_DFL_MAP_ALLOC;
|
/* insert new subblocks */
|
||||||
while (new_alloc < target)
|
|
||||||
new_alloc *= 2;
|
|
||||||
|
|
||||||
new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
|
|
||||||
if (!new)
|
|
||||||
return -ENOMEM;
|
|
||||||
|
|
||||||
size = chunk->map_alloc * sizeof(chunk->map[0]);
|
|
||||||
memcpy(new, chunk->map, size);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* map_alloc < PCPU_DFL_MAP_ALLOC indicates that the
|
|
||||||
* chunk is one of the first chunks and still using
|
|
||||||
* static map.
|
|
||||||
*/
|
|
||||||
if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC)
|
|
||||||
pcpu_mem_free(chunk->map, size);
|
|
||||||
|
|
||||||
chunk->map_alloc = new_alloc;
|
|
||||||
chunk->map = new;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* insert a new subblock */
|
|
||||||
memmove(&chunk->map[i + nr_extra], &chunk->map[i],
|
memmove(&chunk->map[i + nr_extra], &chunk->map[i],
|
||||||
sizeof(chunk->map[0]) * (chunk->map_used - i));
|
sizeof(chunk->map[0]) * (chunk->map_used - i));
|
||||||
chunk->map_used += nr_extra;
|
chunk->map_used += nr_extra;
|
||||||
|
@ -371,7 +387,6 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
|
||||||
chunk->map[i++] -= tail;
|
chunk->map[i++] -= tail;
|
||||||
chunk->map[i] = tail;
|
chunk->map[i] = tail;
|
||||||
}
|
}
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -384,8 +399,11 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
|
||||||
* Note that this function only allocates the offset. It doesn't
|
* Note that this function only allocates the offset. It doesn't
|
||||||
* populate or map the area.
|
* populate or map the area.
|
||||||
*
|
*
|
||||||
|
* @chunk->map must have at least two free slots.
|
||||||
|
*
|
||||||
* RETURNS:
|
* RETURNS:
|
||||||
* Allocated offset in @chunk on success, -errno on failure.
|
* Allocated offset in @chunk on success, -1 if no matching area is
|
||||||
|
* found.
|
||||||
*/
|
*/
|
||||||
static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
|
static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
|
||||||
{
|
{
|
||||||
|
@ -433,8 +451,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
|
||||||
|
|
||||||
/* split if warranted */
|
/* split if warranted */
|
||||||
if (head || tail) {
|
if (head || tail) {
|
||||||
if (pcpu_split_block(chunk, i, head, tail))
|
pcpu_split_block(chunk, i, head, tail);
|
||||||
return -ENOMEM;
|
|
||||||
if (head) {
|
if (head) {
|
||||||
i++;
|
i++;
|
||||||
off += head;
|
off += head;
|
||||||
|
@ -461,14 +478,8 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
|
||||||
chunk->contig_hint = max_contig; /* fully scanned */
|
chunk->contig_hint = max_contig; /* fully scanned */
|
||||||
pcpu_chunk_relocate(chunk, oslot);
|
pcpu_chunk_relocate(chunk, oslot);
|
||||||
|
|
||||||
/*
|
/* tell the upper layer that this chunk has no matching area */
|
||||||
* Tell the upper layer that this chunk has no area left.
|
return -1;
|
||||||
* Note that this is not an error condition but a notification
|
|
||||||
* to upper layer that it needs to look at other chunks.
|
|
||||||
* -ENOSPC is chosen as it isn't used in memory subsystem and
|
|
||||||
* matches the meaning in a way.
|
|
||||||
*/
|
|
||||||
return -ENOSPC;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -755,7 +766,8 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
/* serve reserved allocations from the reserved chunk if available */
|
/* serve reserved allocations from the reserved chunk if available */
|
||||||
if (reserved && pcpu_reserved_chunk) {
|
if (reserved && pcpu_reserved_chunk) {
|
||||||
chunk = pcpu_reserved_chunk;
|
chunk = pcpu_reserved_chunk;
|
||||||
if (size > chunk->contig_hint)
|
if (size > chunk->contig_hint ||
|
||||||
|
pcpu_extend_area_map(chunk) < 0)
|
||||||
goto out_unlock;
|
goto out_unlock;
|
||||||
off = pcpu_alloc_area(chunk, size, align);
|
off = pcpu_alloc_area(chunk, size, align);
|
||||||
if (off >= 0)
|
if (off >= 0)
|
||||||
|
@ -768,11 +780,11 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
list_for_each_entry(chunk, &pcpu_slot[slot], list) {
|
list_for_each_entry(chunk, &pcpu_slot[slot], list) {
|
||||||
if (size > chunk->contig_hint)
|
if (size > chunk->contig_hint)
|
||||||
continue;
|
continue;
|
||||||
|
if (pcpu_extend_area_map(chunk) < 0)
|
||||||
|
goto out_unlock;
|
||||||
off = pcpu_alloc_area(chunk, size, align);
|
off = pcpu_alloc_area(chunk, size, align);
|
||||||
if (off >= 0)
|
if (off >= 0)
|
||||||
goto area_found;
|
goto area_found;
|
||||||
if (off != -ENOSPC)
|
|
||||||
goto out_unlock;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue