mm, thp: do not access mm->pmd_huge_pte directly

Currently mm->pmd_huge_pte protected by page table lock.  It will not
work with split lock.  We have to have per-pmd pmd_huge_pte for proper
access serialization.

For now, let's just introduce wrapper to access mm->pmd_huge_pte.

Signed-off-by: Kirill A. Shutemov <[email protected]>
Tested-by: Alex Thorlton <[email protected]>
Cc: Alex Thorlton <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Naoya Horiguchi <[email protected]>
Cc: "Eric W . Biederman" <[email protected]>
Cc: "Paul E . McKenney" <[email protected]>
Cc: Al Viro <[email protected]>
Cc: Andi Kleen <[email protected]>
Cc: Andrea Arcangeli <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: Dave Jones <[email protected]>
Cc: David Howells <[email protected]>
Cc: Frederic Weisbecker <[email protected]>
Cc: Johannes Weiner <[email protected]>
Cc: Kees Cook <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: Michael Kerrisk <[email protected]>
Cc: Oleg Nesterov <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Robin Holt <[email protected]>
Cc: Sedat Dilek <[email protected]>
Cc: Srikar Dronamraju <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Hugh Dickins <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 0a2e5e086749c00b98bbcba84446377935fd87df..1ea18fcfa2112788fb85c218754bb1d5dd7a2791 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -1244,11 +1244,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
        assert_spin_locked(&mm->page_table_lock);
 
        /* FIFO */
-       if (!mm->pmd_huge_pte)
+       if (!pmd_huge_pte(mm, pmdp))
                INIT_LIST_HEAD(lh);
        else
-               list_add(lh, (struct list_head *) mm->pmd_huge_pte);
-       mm->pmd_huge_pte = pgtable;
+               list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
+       pmd_huge_pte(mm, pmdp) = pgtable;
 }
 
 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
@@ -1260,12 +1260,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
        assert_spin_locked(&mm->page_table_lock);
 
        /* FIFO */
-       pgtable = mm->pmd_huge_pte;
+       pgtable = pmd_huge_pte(mm, pmdp);
        lh = (struct list_head *) pgtable;
        if (list_empty(lh))
-               mm->pmd_huge_pte = NULL;
+               pmd_huge_pte(mm, pmdp) = NULL;
        else {
-               mm->pmd_huge_pte = (pgtable_t) lh->next;
+               pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
                list_del(lh);
        }
        ptep = (pte_t *) pgtable;

diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 7a91f288c7081229a6c2d2a4a1385a9673c58fc5..656cc46a81f578a32d877dce7a3723daf3eaf3eb 100644 (file)
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -196,11 +196,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
        assert_spin_locked(&mm->page_table_lock);
 
        /* FIFO */
-       if (!mm->pmd_huge_pte)
+       if (!pmd_huge_pte(mm, pmdp))
                INIT_LIST_HEAD(lh);
        else
-               list_add(lh, (struct list_head *) mm->pmd_huge_pte);
-       mm->pmd_huge_pte = pgtable;
+               list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
+       pmd_huge_pte(mm, pmdp) = pgtable;
 }
 
 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
@@ -211,12 +211,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
        assert_spin_locked(&mm->page_table_lock);
 
        /* FIFO */
-       pgtable = mm->pmd_huge_pte;
+       pgtable = pmd_huge_pte(mm, pmdp);
        lh = (struct list_head *) pgtable;
        if (list_empty(lh))
-               mm->pmd_huge_pte = NULL;
+               pmd_huge_pte(mm, pmdp) = NULL;
        else {
-               mm->pmd_huge_pte = (pgtable_t) lh->next;
+               pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
                list_del(lh);
        }
        pte_val(pgtable[0]) = 0;

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 4f4ca41fcf273a35eb0a260741ef894b4a4cdb21..861cad53b74469407e7f0186f5c185a7550cba4a 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1383,6 +1383,7 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
        return &mm->page_table_lock;
 }
 
+#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
 
 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
 {

diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c
index 3929a40bd6c0a6d618d0dc0136fe6aecaad08628..41fee3e5d57005943c0cbae7f7a7ddace15e1d04 100644 (file)
--- a/mm/pgtable-generic.c
+++ b/mm/pgtable-generic.c
@@ -154,11 +154,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
        assert_spin_locked(&mm->page_table_lock);
 
        /* FIFO */
-       if (!mm->pmd_huge_pte)
+       if (!pmd_huge_pte(mm, pmdp))
                INIT_LIST_HEAD(&pgtable->lru);
        else
-               list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
-       mm->pmd_huge_pte = pgtable;
+               list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
+       pmd_huge_pte(mm, pmdp) = pgtable;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 #endif
@@ -173,11 +173,11 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
        assert_spin_locked(&mm->page_table_lock);
 
        /* FIFO */
-       pgtable = mm->pmd_huge_pte;
+       pgtable = pmd_huge_pte(mm, pmdp);
        if (list_empty(&pgtable->lru))
-               mm->pmd_huge_pte = NULL;
+               pmd_huge_pte(mm, pmdp) = NULL;
        else {
-               mm->pmd_huge_pte = list_entry(pgtable->lru.next,
+               pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
                                              struct page, lru);
                list_del(&pgtable->lru);
        }