mm/nobootmem.c: have __free_pages_memory() free in larger chunks.

On large memory machines it can take a few minutes to get through
free_all_bootmem().

Currently, when free_all_bootmem() calls __free_pages_memory(), the number
of contiguous pages that __free_pages_memory() passes to the buddy
allocator is limited to BITS_PER_LONG.  BITS_PER_LONG was originally
chosen to keep things similar to mm/nobootmem.c.  But it is more efficient
to limit it to MAX_ORDER.

       base   new  change
8TB    202s  172s   30s
16TB   401s  351s   50s

That is around 1%-3% improvement on total boot time.

This patch was spun off from the boot time rfc Robin and I had been
working on.

Signed-off-by: Robin Holt <[email protected]>
Signed-off-by: Nathan Zimmer <[email protected]>
Cc: Robin Holt <[email protected]>
Cc: "H. Peter Anvin" <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Mike Travis <[email protected]>
Cc: Yinghai Lu <[email protected]>
Cc: Mel Gorman <[email protected]>
Acked-by: Johannes Weiner <[email protected]>
Reviewed-by: Wanpeng Li <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/mm/nobootmem.c b/mm/nobootmem.c
index 61107cf55bb3e2c491bffdfb63f489b9587b6f06..2c254d37465549b0230b543cd9ac7c4d60cdf7c1 100644 (file)
--- a/mm/nobootmem.c
+++ b/mm/nobootmem.c
@@ -82,27 +82,18 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
 
 static void __init __free_pages_memory(unsigned long start, unsigned long end)
 {
-       unsigned long i, start_aligned, end_aligned;
-       int order = ilog2(BITS_PER_LONG);
+       int order;
 
-       start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
-       end_aligned = end & ~(BITS_PER_LONG - 1);
+       while (start < end) {
+               order = min(MAX_ORDER - 1UL, __ffs(start));
 
-       if (end_aligned <= start_aligned) {
-               for (i = start; i < end; i++)
-                       __free_pages_bootmem(pfn_to_page(i), 0);
+               while (start + (1UL << order) > end)
+                       order--;
 
-               return;
-       }
-
-       for (i = start; i < start_aligned; i++)
-               __free_pages_bootmem(pfn_to_page(i), 0);
+               __free_pages_bootmem(pfn_to_page(start), order);
 
-       for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
-               __free_pages_bootmem(pfn_to_page(i), order);
-
-       for (i = end_aligned; i < end; i++)
-               __free_pages_bootmem(pfn_to_page(i), 0);
+               start += (1UL << order);
+       }
 }
 
 static unsigned long __init __free_memory_core(phys_addr_t start,