x86/efi: Add 5-level paging support

Allocate additional page table level and ajdust efi_sync_low_kernel_mappings()
to work with additional page table level.

Signed-off-by: Kirill A. Shutemov <[email protected]>
Reviewed-by: Matt Fleming <[email protected]>
Acked-by: Thomas Gleixner <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Arnd Bergmann <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Brian Gerst <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: Denys Vlasenko <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Josh Poimboeuf <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: [email protected]
Cc: [email protected]
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>

diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 8544dae3d1b4d29607bdf536984684e066c3d0d3..eb64e5b33e3773460fadbd16247eeda6cc318f14 100644 (file)
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -135,6 +135,7 @@ static pgd_t *efi_pgd;
 int __init efi_alloc_page_tables(void)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        gfp_t gfp_mask;
 
@@ -147,15 +148,20 @@ int __init efi_alloc_page_tables(void)
                return -ENOMEM;
 
        pgd = efi_pgd + pgd_index(EFI_VA_END);
+       p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END);
+       if (!p4d) {
+               free_page((unsigned long)efi_pgd);
+               return -ENOMEM;
+       }
 
-       pud = pud_alloc_one(NULL, 0);
+       pud = pud_alloc(&init_mm, p4d, EFI_VA_END);
        if (!pud) {
+               if (CONFIG_PGTABLE_LEVELS > 4)
+                       free_page((unsigned long) pgd_page_vaddr(*pgd));
                free_page((unsigned long)efi_pgd);
                return -ENOMEM;
        }
 
-       pgd_populate(NULL, pgd, pud);
-
        return 0;
 }
 
@@ -190,6 +196,21 @@ void efi_sync_low_kernel_mappings(void)
        num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET);
        memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries);
 
+       /*
+        * As with PGDs, we share all P4D entries apart from the one entry
+        * that covers the EFI runtime mapping space.
+        */
+       BUILD_BUG_ON(p4d_index(EFI_VA_END) != p4d_index(MODULES_END));
+       BUILD_BUG_ON((EFI_VA_START & P4D_MASK) != (EFI_VA_END & P4D_MASK));
+
+       pgd_efi = efi_pgd + pgd_index(EFI_VA_END);
+       pgd_k = pgd_offset_k(EFI_VA_END);
+       p4d_efi = p4d_offset(pgd_efi, 0);
+       p4d_k = p4d_offset(pgd_k, 0);
+
+       num_entries = p4d_index(EFI_VA_END);
+       memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries);
+
        /*
         * We share all the PUD entries apart from those that map the
         * EFI regions. Copy around them.
@@ -197,20 +218,15 @@ void efi_sync_low_kernel_mappings(void)
        BUILD_BUG_ON((EFI_VA_START & ~PUD_MASK) != 0);
        BUILD_BUG_ON((EFI_VA_END & ~PUD_MASK) != 0);
 
-       pgd_efi = efi_pgd + pgd_index(EFI_VA_END);
-       p4d_efi = p4d_offset(pgd_efi, 0);
+       p4d_efi = p4d_offset(pgd_efi, EFI_VA_END);
+       p4d_k = p4d_offset(pgd_k, EFI_VA_END);
        pud_efi = pud_offset(p4d_efi, 0);
-
-       pgd_k = pgd_offset_k(EFI_VA_END);
-       p4d_k = p4d_offset(pgd_k, 0);
        pud_k = pud_offset(p4d_k, 0);
 
        num_entries = pud_index(EFI_VA_END);
        memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);
 
-       p4d_efi = p4d_offset(pgd_efi, EFI_VA_START);
        pud_efi = pud_offset(p4d_efi, EFI_VA_START);
-       p4d_k = p4d_offset(pgd_k, EFI_VA_START);
        pud_k = pud_offset(p4d_k, EFI_VA_START);
 
        num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START);