memcg: clear pc->mem_cgroup if necessary.

This is a preparation before removing a flag PCG_ACCT_LRU in page_cgroup
and reducing atomic ops/complexity in memcg LRU handling.

In some cases, pages are added to lru before charge to memcg and pages
are not classfied to memory cgroup at lru addtion.  Now, the lru where
the page should be added is determined a bit in page_cgroup->flags and
pc->mem_cgroup.  I'd like to remove the check of flag.

To handle the case pc->mem_cgroup may contain stale pointers if pages
are added to LRU before classification.  This patch resets
pc->mem_cgroup to root_mem_cgroup before lru additions.

[[email protected]: fix CONFIG_CGROUP_MEM_CONT=n build]
[[email protected]: fix CONFIG_CGROUP_MEM_RES_CTLR=y CONFIG_CGROUP_MEM_RES_CTLR_SWAP=n build]
[[email protected]: ksm.c needs memcontrol.h, per Michal]
[[email protected]: stop oops in mem_cgroup_reset_owner()]
[[email protected]: fix page migration to reset_owner]
Signed-off-by: KAMEZAWA Hiroyuki <[email protected]>
Cc: Miklos Szeredi <[email protected]>
Acked-by: Michal Hocko <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Ying Han <[email protected]>
Signed-off-by: Hugh Dickins <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index b80de520670b64a67dc2ceb8b4e4e11c1d623071..4d34356fe644ee5dca447071829ae06ab68eeb60 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -129,6 +129,7 @@ extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
 extern void mem_cgroup_replace_page_cache(struct page *oldpage,
                                        struct page *newpage);
 
+extern void mem_cgroup_reset_owner(struct page *page);
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
 extern int do_swap_account;
 #endif
@@ -391,6 +392,10 @@ static inline void mem_cgroup_replace_page_cache(struct page *oldpage,
                                struct page *newpage)
 {
 }
+
+static inline void mem_cgroup_reset_owner(struct page *page)
+{
+}
 #endif /* CONFIG_CGROUP_MEM_CONT */
 
 #if !defined(CONFIG_CGROUP_MEM_RES_CTLR) || !defined(CONFIG_DEBUG_VM)

diff --git a/mm/ksm.c b/mm/ksm.c
index 310544a379ae9c7b886b3b50815e5f3d5a991ba8..1925ffbfb27f00ac3d3d262ce0ea1c8aaa3a117f 100644 (file)
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -28,6 +28,7 @@
 #include <linux/kthread.h>
 #include <linux/wait.h>
 #include <linux/slab.h>
+#include <linux/memcontrol.h>
 #include <linux/rbtree.h>
 #include <linux/memory.h>
 #include <linux/mmu_notifier.h>
@@ -1571,6 +1572,16 @@ struct page *ksm_does_need_to_copy(struct page *page,
 
        new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
        if (new_page) {
+               /*
+                * The memcg-specific accounting when moving
+                * pages around the LRU lists relies on the
+                * page's owner (memcg) to be valid.  Usually,
+                * pages are assigned to a new owner before
+                * being put on the LRU list, but since this
+                * is not the case here, the stale owner from
+                * a previous allocation cycle must be reset.
+                */
+               mem_cgroup_reset_owner(new_page);
                copy_user_highpage(new_page, page, address, vma);
 
                SetPageDirty(new_page);

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index d58bb5fa4403861601b867747984362b4a80c028..c74102d6eb5a3ed1c657d11d5ecba0d473fea93b 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3050,6 +3050,23 @@ void mem_cgroup_uncharge_end(void)
        batch->memcg = NULL;
 }
 
+/*
+ * A function for resetting pc->mem_cgroup for newly allocated pages.
+ * This function should be called if the newpage will be added to LRU
+ * before start accounting.
+ */
+void mem_cgroup_reset_owner(struct page *newpage)
+{
+       struct page_cgroup *pc;
+
+       if (mem_cgroup_disabled())
+               return;
+
+       pc = lookup_page_cgroup(newpage);
+       VM_BUG_ON(PageCgroupUsed(pc));
+       pc->mem_cgroup = root_mem_cgroup;
+}
+
 #ifdef CONFIG_SWAP
 /*
  * called after __delete_from_swap_cache() and drop "page" account.

diff --git a/mm/migrate.c b/mm/migrate.c
index 89ea0854332ec3fdf53123bbf7fe5f1de94f6599..fc391985899f8ae53c2a405a7a16019457665d75 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -777,6 +777,8 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
        if (!newpage)
                return -ENOMEM;
 
+       mem_cgroup_reset_owner(newpage);
+
        if (page_count(page) == 1) {
                /* page was freed from under us. So we are done. */
                goto out;

diff --git a/mm/swap_state.c b/mm/swap_state.c
index ea6b32d61873185f59db1eb9ae303488ec6de471..470038a9187383790751817cda61a07cb50f9512 100644 (file)
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -300,6 +300,16 @@ struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                        new_page = alloc_page_vma(gfp_mask, vma, addr);
                        if (!new_page)
                                break;          /* Out of memory */
+                       /*
+                        * The memcg-specific accounting when moving
+                        * pages around the LRU lists relies on the
+                        * page's owner (memcg) to be valid.  Usually,
+                        * pages are assigned to a new owner before
+                        * being put on the LRU list, but since this
+                        * is not the case here, the stale owner from
+                        * a previous allocation cycle must be reset.
+                        */
+                       mem_cgroup_reset_owner(new_page);
                }
 
                /*