mm: add preempt points into __purge_vmap_area_lazy()

Use cond_resched_lock to avoid holding the vmap_area_lock for a
potentially long time and thus creating bad latencies for various
workloads.

[hch: split from a larger patch by Joel, wrote the crappy changelog]
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Joel Fernandes <[email protected]>
Signed-off-by: Christoph Hellwig <[email protected]>
Tested-by: Jisheng Zhang <[email protected]>
Cc: Andrey Ryabinin <[email protected]>
Cc: Chris Wilson <[email protected]>
Cc: John Dias <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: "H. Peter Anvin" <[email protected]>
Cc: Ingo Molnar <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index d3c1f5ee48b48f95f61d12b0b361aaa4cef260bd..a5584384eabc1faf00352d9773ced1c31a09b984 100644 (file)
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -628,7 +628,7 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end)
        struct llist_node *valist;
        struct vmap_area *va;
        struct vmap_area *n_va;
-       int nr = 0;
+       bool do_free = false;
 
        lockdep_assert_held(&vmap_purge_lock);
 
@@ -638,18 +638,22 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end)
                        start = va->va_start;
                if (va->va_end > end)
                        end = va->va_end;
-               nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
+               do_free = true;
        }
 
-       if (!nr)
+       if (!do_free)
                return false;
 
-       atomic_sub(nr, &vmap_lazy_nr);
        flush_tlb_kernel_range(start, end);
 
        spin_lock(&vmap_area_lock);
-       llist_for_each_entry_safe(va, n_va, valist, purge_list)
+       llist_for_each_entry_safe(va, n_va, valist, purge_list) {
+               int nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
+
                __free_vmap_area(va);
+               atomic_sub(nr, &vmap_lazy_nr);
+               cond_resched_lock(&vmap_area_lock);
+       }
        spin_unlock(&vmap_area_lock);
        return true;
 }