mm: nommu: sort mm->mmap list properly

When I was reading nommu code, I found that it handles the vma list/tree
in an unusual way.  IIUC, because there can be more than one
identical/overrapped vmas in the list/tree, it sorts the tree more
strictly and does a linear search on the tree.  But it doesn't applied to
the list (i.e.  the list could be constructed in a different order than
the tree so that we can't use the list when finding the first vma in that
order).

Since inserting/sorting a vma in the tree and link is done at the same
time, we can easily construct both of them in the same order.  And linear
searching on the tree could be more costly than doing it on the list, it
can be converted to use the list.

Also, after the commit 297c5eee3724 ("mm: make the vma list be doubly
linked") made the list be doubly linked, there were a couple of code need
to be fixed to construct the list properly.

Patch 1/6 is a preparation.  It maintains the list sorted same as the tree
and construct doubly-linked list properly.  Patch 2/6 is a simple
optimization for the vma deletion.  Patch 3/6 and 4/6 convert tree
traversal to list traversal and the rest are simple fixes and cleanups.

This patch:

@vma added into @mm should be sorted by start addr, end addr and VMA
struct addr in that order because we may get identical VMAs in the @mm.
However this was true only for the rbtree, not for the list.

This patch fixes this by remembering 'rb_prev' during the tree traversal
like find_vma_prepare() does and linking the @vma via __vma_link_list().
After this patch, we can iterate the whole VMAs in correct order simply by
using @mm->mmap list.

[[email protected]: avoid duplicating __vma_link_list()]
Signed-off-by: Namhyung Kim <[email protected]>
Acked-by: Greg Ungerer <[email protected]>
Cc: David Howells <[email protected]>
Cc: Paul Mundt <[email protected]>
Cc: Geert Uytterhoeven <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/mm/internal.h b/mm/internal.h
index 9d0ced8e505e9276dabb42e129a4b918966a2421..d071d380fb498ab36ac6700343bf86485463bdbe 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -66,6 +66,10 @@ static inline unsigned long page_order(struct page *page)
        return page_private(page);
 }
 
+/* mm/util.c */
+void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
+               struct vm_area_struct *prev, struct rb_node *rb_parent);
+
 #ifdef CONFIG_MMU
 extern long mlock_vma_pages_range(struct vm_area_struct *vma,
                        unsigned long start, unsigned long end);

diff --git a/mm/mmap.c b/mm/mmap.c
index 4fb5464f7707e795eefd00834158941406021e9f..e76f8d752884995efd97704b84db9bf5dd35aaa7 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -398,29 +398,6 @@ find_vma_prepare(struct mm_struct *mm, unsigned long addr,
        return vma;
 }
 
-static inline void
-__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
-               struct vm_area_struct *prev, struct rb_node *rb_parent)
-{
-       struct vm_area_struct *next;
-
-       vma->vm_prev = prev;
-       if (prev) {
-               next = prev->vm_next;
-               prev->vm_next = vma;
-       } else {
-               mm->mmap = vma;
-               if (rb_parent)
-                       next = rb_entry(rb_parent,
-                                       struct vm_area_struct, vm_rb);
-               else
-                       next = NULL;
-       }
-       vma->vm_next = next;
-       if (next)
-               next->vm_prev = vma;
-}
-
 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
                struct rb_node **rb_link, struct rb_node *rb_parent)
 {

diff --git a/mm/nommu.c b/mm/nommu.c
index 5afce48ce33937c6e0bd553402684d16bfa81afa..0b16cb4c517bfbdc9655f698e37f89d3f0bceec6 100644 (file)
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -680,9 +680,9 @@ static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
  */
 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
 {
-       struct vm_area_struct *pvma, **pp, *next;
+       struct vm_area_struct *pvma, *prev;
        struct address_space *mapping;
-       struct rb_node **p, *parent;
+       struct rb_node **p, *parent, *rb_prev;
 
        kenter(",%p", vma);
 
@@ -703,7 +703,7 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
        }
 
        /* add the VMA to the tree */
-       parent = NULL;
+       parent = rb_prev = NULL;
        p = &mm->mm_rb.rb_node;
        while (*p) {
                parent = *p;
@@ -713,17 +713,20 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
                 * (the latter is necessary as we may get identical VMAs) */
                if (vma->vm_start < pvma->vm_start)
                        p = &(*p)->rb_left;
-               else if (vma->vm_start > pvma->vm_start)
+               else if (vma->vm_start > pvma->vm_start) {
+                       rb_prev = parent;
                        p = &(*p)->rb_right;
-               else if (vma->vm_end < pvma->vm_end)
+               } else if (vma->vm_end < pvma->vm_end)
                        p = &(*p)->rb_left;
-               else if (vma->vm_end > pvma->vm_end)
+               else if (vma->vm_end > pvma->vm_end) {
+                       rb_prev = parent;
                        p = &(*p)->rb_right;
-               else if (vma < pvma)
+               } else if (vma < pvma)
                        p = &(*p)->rb_left;
-               else if (vma > pvma)
+               else if (vma > pvma) {
+                       rb_prev = parent;
                        p = &(*p)->rb_right;
-               else
+               } else
                        BUG();
        }
 
@@ -731,20 +734,11 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
        rb_insert_color(&vma->vm_rb, &mm->mm_rb);
 
        /* add VMA to the VMA list also */
-       for (pp = &mm->mmap; (pvma = *pp); pp = &(*pp)->vm_next) {
-               if (pvma->vm_start > vma->vm_start)
-                       break;
-               if (pvma->vm_start < vma->vm_start)
-                       continue;
-               if (pvma->vm_end < vma->vm_end)
-                       break;
-       }
+       prev = NULL;
+       if (rb_prev)
+               prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
 
-       next = *pp;
-       *pp = vma;
-       vma->vm_next = next;
-       if (next)
-               next->vm_prev = vma;
+       __vma_link_list(mm, vma, prev, parent);
 }
 
 /*

diff --git a/mm/util.c b/mm/util.c
index e7b103a6fd21c50857eda25cb5750e6d319e6498..88ea1bd661c0f5164e37a8415050808fa815808f 100644 (file)
--- a/mm/util.c
+++ b/mm/util.c
@@ -6,6 +6,8 @@
 #include <linux/sched.h>
 #include <asm/uaccess.h>
 
+#include "internal.h"
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/kmem.h>
 
@@ -215,6 +217,28 @@ char *strndup_user(const char __user *s, long n)
 }
 EXPORT_SYMBOL(strndup_user);
 
+void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
+               struct vm_area_struct *prev, struct rb_node *rb_parent)
+{
+       struct vm_area_struct *next;
+
+       vma->vm_prev = prev;
+       if (prev) {
+               next = prev->vm_next;
+               prev->vm_next = vma;
+       } else {
+               mm->mmap = vma;
+               if (rb_parent)
+                       next = rb_entry(rb_parent,
+                                       struct vm_area_struct, vm_rb);
+               else
+                       next = NULL;
+       }
+       vma->vm_next = next;
+       if (next)
+               next->vm_prev = vma;
+}
+
 #if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
 void arch_pick_mmap_layout(struct mm_struct *mm)
 {