mm: fix alloc_bootmem_core to use fast searching for all nodes
Make the nodes other than node 0 use bdata->last_success for fast search too. We need to use __alloc_bootmem_core() for vmemmap allocation for other nodes when numa and sparsemem/vmemmap are enabled. Also, make fail_block path increase i with incr only after ALIGN to avoid extra increase when size is larger than align. Signed-off-by: Yinghai Lu <yhlu.kernel@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
parent
e123dd3f0e
commit
ad09315cad
16
mm/bootmem.c
16
mm/bootmem.c
|
@ -238,28 +238,32 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
|
||||||
* We try to allocate bootmem pages above 'goal'
|
* We try to allocate bootmem pages above 'goal'
|
||||||
* first, then we try to allocate lower pages.
|
* first, then we try to allocate lower pages.
|
||||||
*/
|
*/
|
||||||
if (goal && goal >= bdata->node_boot_start && PFN_DOWN(goal) < end_pfn) {
|
preferred = 0;
|
||||||
|
if (goal && PFN_DOWN(goal) < end_pfn) {
|
||||||
|
if (goal > bdata->node_boot_start)
|
||||||
preferred = goal - bdata->node_boot_start;
|
preferred = goal - bdata->node_boot_start;
|
||||||
|
|
||||||
if (bdata->last_success >= preferred)
|
if (bdata->last_success >= preferred)
|
||||||
if (!limit || (limit && limit > bdata->last_success))
|
if (!limit || (limit && limit > bdata->last_success))
|
||||||
preferred = bdata->last_success;
|
preferred = bdata->last_success;
|
||||||
} else
|
}
|
||||||
preferred = 0;
|
|
||||||
|
|
||||||
preferred = PFN_DOWN(ALIGN(preferred, align)) + offset;
|
preferred = PFN_DOWN(ALIGN(preferred, align)) + offset;
|
||||||
areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
|
areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
|
||||||
incr = align >> PAGE_SHIFT ? : 1;
|
incr = align >> PAGE_SHIFT ? : 1;
|
||||||
|
|
||||||
restart_scan:
|
restart_scan:
|
||||||
for (i = preferred; i < eidx; i += incr) {
|
for (i = preferred; i < eidx;) {
|
||||||
unsigned long j;
|
unsigned long j;
|
||||||
|
|
||||||
i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i);
|
i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i);
|
||||||
i = ALIGN(i, incr);
|
i = ALIGN(i, incr);
|
||||||
if (i >= eidx)
|
if (i >= eidx)
|
||||||
break;
|
break;
|
||||||
if (test_bit(i, bdata->node_bootmem_map))
|
if (test_bit(i, bdata->node_bootmem_map)) {
|
||||||
|
i += incr;
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
for (j = i + 1; j < i + areasize; ++j) {
|
for (j = i + 1; j < i + areasize; ++j) {
|
||||||
if (j >= eidx)
|
if (j >= eidx)
|
||||||
goto fail_block;
|
goto fail_block;
|
||||||
|
@ -270,6 +274,8 @@ restart_scan:
|
||||||
goto found;
|
goto found;
|
||||||
fail_block:
|
fail_block:
|
||||||
i = ALIGN(j, incr);
|
i = ALIGN(j, incr);
|
||||||
|
if (i == j)
|
||||||
|
i += incr;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (preferred > offset) {
|
if (preferred > offset) {
|
||||||
|
|
Loading…
Reference in New Issue