1 files changed, 35 insertions, 18 deletions

diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index fd594d5fe142..fb9c5a85aa56 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -73,31 +73,28 @@ static void __init_refok *vmem_alloc_pages(unsigned int order)
         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
 }
 
+#define vmem_pud_alloc()        ({ BUG(); ((pud_t *) NULL); })
+
 static inline pmd_t *vmem_pmd_alloc(void)
 {
-        pmd_t *pmd;
-        int i;
+        pmd_t *pmd = NULL;
 
-        pmd = vmem_alloc_pages(PMD_ALLOC_ORDER);
+#ifdef CONFIG_64BIT
+        pmd = vmem_alloc_pages(2);
         if (!pmd)
                 return NULL;
-        for (i = 0; i < PTRS_PER_PMD; i++)
-                pmd_clear_kernel(pmd + i);
+        clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE*4);
+#endif
         return pmd;
 }
 
 static inline pte_t *vmem_pte_alloc(void)
 {
-        pte_t *pte;
-        pte_t empty_pte;
-        int i;
+        pte_t *pte = vmem_alloc_pages(0);
 
-        pte = vmem_alloc_pages(PTE_ALLOC_ORDER);
         if (!pte)
                 return NULL;
-        pte_val(empty_pte) = _PAGE_TYPE_EMPTY;
-        for (i = 0; i < PTRS_PER_PTE; i++)
-                pte[i] = empty_pte;
+        clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, PAGE_SIZE);
         return pte;
 }
 
@@ -108,6 +105,7 @@ static int vmem_add_range(unsigned long start, unsigned long size)
 {
         unsigned long address;
         pgd_t *pg_dir;
+        pud_t *pu_dir;
         pmd_t *pm_dir;
         pte_t *pt_dir;
         pte_t  pte;
@@ -116,13 +114,21 @@ static int vmem_add_range(unsigned long start, unsigned long size)
         for (address = start; address < start + size; address += PAGE_SIZE) {
                 pg_dir = pgd_offset_k(address);
                 if (pgd_none(*pg_dir)) {
+                        pu_dir = vmem_pud_alloc();
+                        if (!pu_dir)
+                                goto out;
+                        pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
+                }
+
+                pu_dir = pud_offset(pg_dir, address);
+                if (pud_none(*pu_dir)) {
                         pm_dir = vmem_pmd_alloc();
                         if (!pm_dir)
                                 goto out;
-                        pgd_populate_kernel(&init_mm, pg_dir, pm_dir);
+                        pud_populate_kernel(&init_mm, pu_dir, pm_dir);
                 }
 
-                pm_dir = pmd_offset(pg_dir, address);
+                pm_dir = pmd_offset(pu_dir, address);
                 if (pmd_none(*pm_dir)) {
                         pt_dir = vmem_pte_alloc();
                         if (!pt_dir)
@@ -148,6 +154,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
 {
         unsigned long address;
         pgd_t *pg_dir;
+        pud_t *pu_dir;
         pmd_t *pm_dir;
         pte_t *pt_dir;
         pte_t  pte;
@@ -155,9 +162,10 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
         pte_val(pte) = _PAGE_TYPE_EMPTY;
         for (address = start; address < start + size; address += PAGE_SIZE) {
                 pg_dir = pgd_offset_k(address);
-                if (pgd_none(*pg_dir))
+                pu_dir = pud_offset(pg_dir, address);
+                if (pud_none(*pu_dir))
                         continue;
-                pm_dir = pmd_offset(pg_dir, address);
+                pm_dir = pmd_offset(pu_dir, address);
                 if (pmd_none(*pm_dir))
                         continue;
                 pt_dir = pte_offset_kernel(pm_dir, address);
@@ -174,6 +182,7 @@ static int vmem_add_mem_map(unsigned long start, unsigned long size)
         unsigned long address, start_addr, end_addr;
         struct page *map_start, *map_end;
         pgd_t *pg_dir;
+        pud_t *pu_dir;
         pmd_t *pm_dir;
         pte_t *pt_dir;
         pte_t  pte;
@@ -188,13 +197,21 @@ static int vmem_add_mem_map(unsigned long start, unsigned long size)
         for (address = start_addr; address < end_addr; address += PAGE_SIZE) {
                 pg_dir = pgd_offset_k(address);
                 if (pgd_none(*pg_dir)) {
+                        pu_dir = vmem_pud_alloc();
+                        if (!pu_dir)
+                                goto out;
+                        pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
+                }
+
+                pu_dir = pud_offset(pg_dir, address);
+                if (pud_none(*pu_dir)) {
                         pm_dir = vmem_pmd_alloc();
                         if (!pm_dir)
                                 goto out;
-                        pgd_populate_kernel(&init_mm, pg_dir, pm_dir);
+                        pud_populate_kernel(&init_mm, pu_dir, pm_dir);
                 }
 
-                pm_dir = pmd_offset(pg_dir, address);
+                pm_dir = pmd_offset(pu_dir, address);
                 if (pmd_none(*pm_dir)) {
                         pt_dir = vmem_pte_alloc();
                         if (!pt_dir)