hugetlbfs: add missing TLB flush to hugetlb_cow()

A cow break on a hugetlbfs page with page_count > 1 will set a new pte with
set_huge_pte_at(), w/o any tlb flush operation.  The old pte will remain in
the tlb and subsequent write access to the page will result in a page fault
loop, for as long as it may take until the tlb is flushed from somewhere else.
 This patch introduces an architecture-specific huge_ptep_clear_flush()
function, which is called before the the set_huge_pte_at() in hugetlb_cow().

ATTENTION: This is just a nop on all architectures for now, the s390
implementation will come with our large page patch later.  Other architectures
should define their own huge_ptep_clear_flush() if needed.

Acked-by: Martin Schwidefsky <[email protected]>
Signed-off-by: Gerald Schaefer <[email protected]>
Cc: Paul Mundt <[email protected]>
Cc: "Luck, Tony" <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: "David S. Miller" <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/include/asm-ia64/hugetlb.h b/include/asm-ia64/hugetlb.h
index f0ee14c6e17253efd3d1f0704bf2e102eab94430..5f543437497257e6a70df013c347d5f11f94fad2 100644 (file)
--- a/include/asm-ia64/hugetlb.h
+++ b/include/asm-ia64/hugetlb.h
@@ -34,4 +34,9 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
        return ptep_get_and_clear(mm, addr, ptep);
 }
 
+static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep)
+{
+}
+
 #endif /* _ASM_IA64_HUGETLB_H */

diff --git a/include/asm-powerpc/hugetlb.h b/include/asm-powerpc/hugetlb.h
index f537993c5c87b31b1d75422b29700f4ab2ac6331..bead2ff78493df8490f1488a3b321fff68d21c66 100644 (file)
--- a/include/asm-powerpc/hugetlb.h
+++ b/include/asm-powerpc/hugetlb.h
@@ -34,4 +34,9 @@ static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm)
 {
 }
 
+static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep)
+{
+}
+
 #endif /* _ASM_POWERPC_HUGETLB_H */

diff --git a/include/asm-sh/hugetlb.h b/include/asm-sh/hugetlb.h
index 885218d2c844c99405935b2c0de527d08bf8ef37..d1ed476467a1989a0f86f4a1abc1e19d050f558f 100644 (file)
--- a/include/asm-sh/hugetlb.h
+++ b/include/asm-sh/hugetlb.h
@@ -46,4 +46,9 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
        return ptep_get_and_clear(mm, addr, ptep);
 }
 
+static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep)
+{
+}
+
 #endif /* _ASM_SH_HUGETLB_H */

diff --git a/include/asm-sparc64/hugetlb.h b/include/asm-sparc64/hugetlb.h
index 7e111cfd31ea184b8583b8773fc542d0bdac576b..0b9e44c85c5da821e7c88a8cd2f04336f7bc2584 100644 (file)
--- a/include/asm-sparc64/hugetlb.h
+++ b/include/asm-sparc64/hugetlb.h
@@ -39,4 +39,9 @@ static inline void hugetlb_free_pgd_range(struct mmu_gather **tlb,
        free_pgd_range(tlb, addr, end, floor, ceiling);
 }
 
+static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep)
+{
+}
+
 #endif /* _ASM_SPARC64_HUGETLB_H */

diff --git a/include/asm-x86/hugetlb.h b/include/asm-x86/hugetlb.h
index ec21cedd71497cb3788183295a1d238581a08dec..f57236dfc8f4b007b921d6191435bb3345e3e7c3 100644 (file)
--- a/include/asm-x86/hugetlb.h
+++ b/include/asm-x86/hugetlb.h
@@ -46,4 +46,9 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
        return ptep_get_and_clear(mm, addr, ptep);
 }
 
+static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep)
+{
+}
+
 #endif /* _ASM_X86_HUGETLB_H */

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index d36e1f11a5f267d441ffda665d4b6a6275c3d451..262d0a93d2b6d45112975357374496bf29d3138b 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -892,6 +892,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
        ptep = huge_pte_offset(mm, address & HPAGE_MASK);
        if (likely(pte_same(*ptep, pte))) {
                /* Break COW */
+               huge_ptep_clear_flush(vma, address, ptep);
                set_huge_pte_at(mm, address, ptep,
                                make_huge_pte(vma, new_page, 1));
                /* Make the old page be freed below */