x86/kexec: Add 5-level paging support

Handle additional page table level in the kexec code.

Signed-off-by: Kirill A. Shutemov <[email protected]>
Acked-by: Thomas Gleixner <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Arnd Bergmann <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Brian Gerst <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: Denys Vlasenko <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Josh Poimboeuf <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>

diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h
index 282630e4c6ea4696e54c6ea650751d913ce49c11..70ef205489f00e53ff568180c4dcbf6fb9e6ded1 100644 (file)
--- a/arch/x86/include/asm/kexec.h
+++ b/arch/x86/include/asm/kexec.h
@@ -164,6 +164,7 @@ struct kimage_arch {
 };
 #else
 struct kimage_arch {
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;

diff --git a/arch/x86/kernel/machine_kexec_32.c b/arch/x86/kernel/machine_kexec_32.c
index 469b23d6acc272b2113878182582d9fa7532f189..5f43cec296c5c38dc28a2a0c0e43aee91ee89934 100644 (file)
--- a/arch/x86/kernel/machine_kexec_32.c
+++ b/arch/x86/kernel/machine_kexec_32.c
@@ -103,6 +103,7 @@ static void machine_kexec_page_table_set_one(
        pgd_t *pgd, pmd_t *pmd, pte_t *pte,
        unsigned long vaddr, unsigned long paddr)
 {
+       p4d_t *p4d;
        pud_t *pud;
 
        pgd += pgd_index(vaddr);
@@ -110,7 +111,8 @@ static void machine_kexec_page_table_set_one(
        if (!(pgd_val(*pgd) & _PAGE_PRESENT))
                set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT));
 #endif
-       pud = pud_offset(pgd, vaddr);
+       p4d = p4d_offset(pgd, vaddr);
+       pud = pud_offset(p4d, vaddr);
        pmd = pmd_offset(pud, vaddr);
        if (!(pmd_val(*pmd) & _PAGE_PRESENT))
                set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));

diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 857cdbd028675716afad71c0b48974399889e622..085c3b300d32e15faeb297028ad45ddb14513dd3 100644 (file)
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -36,6 +36,7 @@ static struct kexec_file_ops *kexec_file_loaders[] = {
 
 static void free_transition_pgtable(struct kimage *image)
 {
+       free_page((unsigned long)image->arch.p4d);
        free_page((unsigned long)image->arch.pud);
        free_page((unsigned long)image->arch.pmd);
        free_page((unsigned long)image->arch.pte);
@@ -43,6 +44,7 @@ static void free_transition_pgtable(struct kimage *image)
 
 static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
 {
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -53,13 +55,21 @@ static int init_transition_pgtable(struct kimage *image, pgd_t *pgd)
        paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE);
        pgd += pgd_index(vaddr);
        if (!pgd_present(*pgd)) {
+               p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
+               if (!p4d)
+                       goto err;
+               image->arch.p4d = p4d;
+               set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
+       }
+       p4d = p4d_offset(pgd, vaddr);
+       if (!p4d_present(*p4d)) {
                pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
                if (!pud)
                        goto err;
                image->arch.pud = pud;
-               set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
+               set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
        }
-       pud = pud_offset(pgd, vaddr);
+       pud = pud_offset(p4d, vaddr);
        if (!pud_present(*pud)) {
                pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
                if (!pmd)