[S390] System z large page support.

This adds hugetlbfs support on System z, using both hardware large page
support if available and software large page emulation on older hardware.
Shared (large) page tables are implemented in software emulation mode,
by using page->index of the first tail page from a compound large page
to store page table information.

Signed-off-by: Gerald Schaefer <geraldsc@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

8 files changed, 210 insertions, 35 deletions

diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index bd188f6bd0e2..d0e09684b9ce 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -268,6 +268,19 @@ static noinline __init void setup_lowcore_early(void)
         s390_base_pgm_handler_fn = early_pgm_check_handler;
 }
 
+static noinline __init void setup_hpage(void)
+{
+#ifndef CONFIG_DEBUG_PAGEALLOC
+        unsigned int facilities;
+
+        facilities = stfl();
+        if (!(facilities & (1UL << 23)) || !(facilities & (1UL << 29)))
+                return;
+        machine_flags |= MACHINE_FLAG_HPAGE;
+        __ctl_set_bit(0, 23);
+#endif
+}
+
 static __init void detect_mvpg(void)
 {
 #ifndef CONFIG_64BIT
@@ -360,6 +373,8 @@ static __init void detect_machine_facilities(void)
         facilities = stfl();
         if (facilities & (1 << 28))
                 machine_flags |= MACHINE_FLAG_IDTE;
+        if (facilities & (1 << 23))
+                machine_flags |= MACHINE_FLAG_PFMF;
         if (facilities & (1 << 4))
                 machine_flags |= MACHINE_FLAG_MVCOS;
 #endif
@@ -388,6 +403,7 @@ void __init startup_init(void)
         detect_diag9c();
         detect_diag44();
         detect_machine_facilities();
+        setup_hpage();
         sclp_read_info_early();
         sclp_facilities_detect();
         memsize = sclp_memory_detect();
diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S
index 9c2c6f7d37e7..1d06961e87b3 100644
--- a/arch/s390/kernel/head64.S
+++ b/arch/s390/kernel/head64.S
@@ -129,7 +129,7 @@ startup_continue:
                                         # virtual and never return ...
         .align  16
 .Lentry:.quad   0x0000000180000000,_stext
-.Lctl:  .quad   0x04b50002              # cr0: various things
+.Lctl:  .quad   0x04350002              # cr0: various things
         .quad   0                       # cr1: primary space segment table
         .quad   .Lduct                  # cr2: dispatchable unit control table
         .quad   0                       # cr3: instruction authorization
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 694c6546ce64..2bc70b6e876a 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -749,6 +749,9 @@ static void __init setup_hwcaps(void)
                         elf_hwcap |= 1UL << 6;
         }
 
+        if (MACHINE_HAS_HPAGE)
+                elf_hwcap |= 1UL << 7;
+
         switch (cpuinfo->cpu_id.machine) {
         case 0x9672:
 #if !defined(CONFIG_64BIT)
@@ -872,8 +875,9 @@ void __cpuinit print_cpu_info(struct cpuinfo_S390 *cpuinfo)
 
 static int show_cpuinfo(struct seq_file *m, void *v)
 {
-        static const char *hwcap_str[7] = {
-                "esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp"
+        static const char *hwcap_str[8] = {
+                "esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp",
+                "edat"
         };
         struct cpuinfo_S390 *cpuinfo;
         unsigned long n = (unsigned long) v - 1;
@@ -888,7 +892,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
                                num_online_cpus(), loops_per_jiffy/(500000/HZ),
                                (loops_per_jiffy/(5000/HZ))%100);
                 seq_puts(m, "features\t: ");
-                for (i = 0; i < 7; i++)
+                for (i = 0; i < 8; i++)
                         if (hwcap_str[i] && (elf_hwcap & (1UL << i)))
                                 seq_printf(m, "%s ", hwcap_str[i]);
                 seq_puts(m, "\n");
diff --git a/arch/s390/mm/Makefile b/arch/s390/mm/Makefile
index 66401930f83e..fb988a48a754 100644
--- a/arch/s390/mm/Makefile
+++ b/arch/s390/mm/Makefile
@@ -4,4 +4,4 @@
 
 obj-y    := init.o fault.o extmem.o mmap.o vmem.o pgtable.o
 obj-$(CONFIG_CMM) += cmm.o
-
+obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 2650f46001d0..4d537205e83c 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -28,6 +28,7 @@
 #include <linux/hardirq.h>
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
+#include <linux/hugetlb.h>
 #include <asm/system.h>
 #include <asm/pgtable.h>
 #include <asm/s390_ext.h>
@@ -367,6 +368,8 @@ good_area:
         }
 
 survive:
+        if (is_vm_hugetlb_page(vma))
+                address &= HPAGE_MASK;
         /*
          * If for any reason at all we couldn't handle the fault,
          * make sure we exit gracefully rather than endlessly redo
diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
new file mode 100644
index 000000000000..f4b6124fdb75
--- /dev/null
+++ b/arch/s390/mm/hugetlbpage.c
@@ -0,0 +1,134 @@
+/*
+ *  IBM System z Huge TLB Page Support for Kernel.
+ *
+ *    Copyright 2007 IBM Corp.
+ *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/hugetlb.h>
+
+
+void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
+                                   pte_t *pteptr, pte_t pteval)
+{
+        pmd_t *pmdp = (pmd_t *) pteptr;
+        pte_t shadow_pteval = pteval;
+        unsigned long mask;
+
+        if (!MACHINE_HAS_HPAGE) {
+                pteptr = (pte_t *) pte_page(pteval)[1].index;
+                mask = pte_val(pteval) &
+                                (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO);
+                pte_val(pteval) = (_SEGMENT_ENTRY + __pa(pteptr)) | mask;
+                if (mm->context.noexec) {
+                        pteptr += PTRS_PER_PTE;
+                        pte_val(shadow_pteval) =
+                                        (_SEGMENT_ENTRY + __pa(pteptr)) | mask;
+                }
+        }
+
+        pmd_val(*pmdp) = pte_val(pteval);
+        if (mm->context.noexec) {
+                pmdp = get_shadow_table(pmdp);
+                pmd_val(*pmdp) = pte_val(shadow_pteval);
+        }
+}
+
+int arch_prepare_hugepage(struct page *page)
+{
+        unsigned long addr = page_to_phys(page);
+        pte_t pte;
+        pte_t *ptep;
+        int i;
+
+        if (MACHINE_HAS_HPAGE)
+                return 0;
+
+        ptep = (pte_t *) pte_alloc_one(&init_mm, address);
+        if (!ptep)
+                return -ENOMEM;
+
+        pte = mk_pte(page, PAGE_RW);
+        for (i = 0; i < PTRS_PER_PTE; i++) {
+                set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte);
+                pte_val(pte) += PAGE_SIZE;
+        }
+        page[1].index = (unsigned long) ptep;
+        return 0;
+}
+
+void arch_release_hugepage(struct page *page)
+{
+        pte_t *ptep;
+
+        if (MACHINE_HAS_HPAGE)
+                return;
+
+        ptep = (pte_t *) page[1].index;
+        if (!ptep)
+                return;
+        pte_free(&init_mm, ptep);
+        page[1].index = 0;
+}
+
+pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
+{
+        pgd_t *pgdp;
+        pud_t *pudp;
+        pmd_t *pmdp = NULL;
+
+        pgdp = pgd_offset(mm, addr);
+        pudp = pud_alloc(mm, pgdp, addr);
+        if (pudp)
+                pmdp = pmd_alloc(mm, pudp, addr);
+        return (pte_t *) pmdp;
+}
+
+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
+{
+        pgd_t *pgdp;
+        pud_t *pudp;
+        pmd_t *pmdp = NULL;
+
+        pgdp = pgd_offset(mm, addr);
+        if (pgd_present(*pgdp)) {
+                pudp = pud_offset(pgdp, addr);
+                if (pud_present(*pudp))
+                        pmdp = pmd_offset(pudp, addr);
+        }
+        return (pte_t *) pmdp;
+}
+
+int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
+{
+        return 0;
+}
+
+struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
+                              int write)
+{
+        return ERR_PTR(-EINVAL);
+}
+
+int pmd_huge(pmd_t pmd)
+{
+        if (!MACHINE_HAS_HPAGE)
+                return 0;
+
+        return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
+}
+
+struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
+                             pmd_t *pmdp, int write)
+{
+        struct page *page;
+
+        if (!MACHINE_HAS_HPAGE)
+                return NULL;
+
+        page = pmd_page(*pmdp);
+        if (page)
+                page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
+        return page;
+}
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 202c952a29b4..acc92f46a096 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -77,28 +77,6 @@ void show_mem(void)
         printk("%lu pages pagetables\n", global_page_state(NR_PAGETABLE));
 }
 
-static void __init setup_ro_region(void)
-{
-        pgd_t *pgd;
-        pud_t *pud;
-        pmd_t *pmd;
-        pte_t *pte;
-        pte_t new_pte;
-        unsigned long address, end;
-
-        address = ((unsigned long)&_stext) & PAGE_MASK;
-        end = PFN_ALIGN((unsigned long)&_eshared);
-
-        for (; address < end; address += PAGE_SIZE) {
-                pgd = pgd_offset_k(address);
-                pud = pud_offset(pgd, address);
-                pmd = pmd_offset(pud, address);
-                pte = pte_offset_kernel(pmd, address);
-                new_pte = mk_pte_phys(address, __pgprot(_PAGE_RO));
-                *pte = new_pte;
-        }
-}
-
 /*
  * paging_init() sets up the page tables
  */
@@ -121,7 +99,6 @@ void __init paging_init(void)
         clear_table((unsigned long *) init_mm.pgd, pgd_type,
                     sizeof(unsigned long)*2048);
         vmem_map_init();
-        setup_ro_region();
 
         /* enable virtual mapping in kernel mode */
         __ctl_load(S390_lowcore.kernel_asce, 1, 1);
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 3ffc0211dc85..97bce6c97574 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -10,10 +10,12 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/list.h>
+#include <linux/hugetlb.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 #include <asm/setup.h>
 #include <asm/tlbflush.h>
+#include <asm/sections.h>
 
 static DEFINE_MUTEX(vmem_mutex);
 
@@ -113,7 +115,7 @@ static pte_t __init_refok *vmem_pte_alloc(void)
 /*
  * Add a physical memory range to the 1:1 mapping.
  */
-static int vmem_add_range(unsigned long start, unsigned long size)
+static int vmem_add_range(unsigned long start, unsigned long size, int ro)
 {
         unsigned long address;
         pgd_t *pg_dir;
@@ -140,7 +142,19 @@ static int vmem_add_range(unsigned long start, unsigned long size)
                         pud_populate_kernel(&init_mm, pu_dir, pm_dir);
                 }
 
+                pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0));
                 pm_dir = pmd_offset(pu_dir, address);
+
+#ifdef __s390x__
+                if (MACHINE_HAS_HPAGE && !(address & ~HPAGE_MASK) &&
+                    (address + HPAGE_SIZE <= start + size) &&
+                    (address >= HPAGE_SIZE)) {
+                        pte_val(pte) |= _SEGMENT_ENTRY_LARGE;
+                        pmd_val(*pm_dir) = pte_val(pte);
+                        address += HPAGE_SIZE - PAGE_SIZE;
+                        continue;
+                }
+#endif
                 if (pmd_none(*pm_dir)) {
                         pt_dir = vmem_pte_alloc();
                         if (!pt_dir)
@@ -149,7 +163,6 @@ static int vmem_add_range(unsigned long start, unsigned long size)
                 }
 
                 pt_dir = pte_offset_kernel(pm_dir, address);
-                pte = pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL);
                 *pt_dir = pte;
         }
         ret = 0;
@@ -180,6 +193,13 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
                 pm_dir = pmd_offset(pu_dir, address);
                 if (pmd_none(*pm_dir))
                         continue;
+
+                if (pmd_huge(*pm_dir)) {
+                        pmd_clear_kernel(pm_dir);
+                        address += HPAGE_SIZE - PAGE_SIZE;
+                        continue;
+                }
+
                 pt_dir = pte_offset_kernel(pm_dir, address);
                 *pt_dir = pte;
         }
@@ -248,14 +268,14 @@ out:
         return ret;
 }
 
-static int vmem_add_mem(unsigned long start, unsigned long size)
+static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
 {
         int ret;
 
         ret = vmem_add_mem_map(start, size);
         if (ret)
                 return ret;
-        return vmem_add_range(start, size);
+        return vmem_add_range(start, size, ro);
 }
 
 /*
@@ -338,7 +358,7 @@ int add_shared_memory(unsigned long start, unsigned long size)
         if (ret)
                 goto out_free;
 
-        ret = vmem_add_mem(start, size);
+        ret = vmem_add_mem(start, size, 0);
         if (ret)
                 goto out_remove;
 
@@ -374,14 +394,35 @@ out:
  */
 void __init vmem_map_init(void)
 {
+        unsigned long ro_start, ro_end;
+        unsigned long start, end;
         int i;
 
         INIT_LIST_HEAD(&init_mm.context.crst_list);
         INIT_LIST_HEAD(&init_mm.context.pgtable_list);
         init_mm.context.noexec = 0;
         NODE_DATA(0)->node_mem_map = VMEM_MAP;
-        for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++)
-                vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size);
+        ro_start = ((unsigned long)&_stext) & PAGE_MASK;
+        ro_end = PFN_ALIGN((unsigned long)&_eshared);
+        for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
+                start = memory_chunk[i].addr;
+                end = memory_chunk[i].addr + memory_chunk[i].size;
+                if (start >= ro_end || end <= ro_start)
+                        vmem_add_mem(start, end - start, 0);
+                else if (start >= ro_start && end <= ro_end)
+                        vmem_add_mem(start, end - start, 1);
+                else if (start >= ro_start) {
+                        vmem_add_mem(start, ro_end - start, 1);
+                        vmem_add_mem(ro_end, end - ro_end, 0);
+                } else if (end < ro_end) {
+                        vmem_add_mem(start, ro_start - start, 0);
+                        vmem_add_mem(ro_start, end - ro_start, 1);
+                } else {
+                        vmem_add_mem(start, ro_start - start, 0);
+                        vmem_add_mem(ro_start, ro_end - ro_start, 1);
+                        vmem_add_mem(ro_end, end - ro_end, 0);
+                }
+        }
 }
 
 /*