9 files changed, 303 insertions, 160 deletions

diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 992bcfff7913..ce4dfa8b834d 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -50,6 +50,10 @@ config SCHED_NO_NO_OMIT_FRAME_POINTER
         bool
         default y
 
+config IA64_UNCACHED_ALLOCATOR
+        bool
+        select GENERIC_ALLOCATOR
+
 choice
         prompt "System type"
         default IA64_GENERIC
@@ -223,7 +227,7 @@ config IA64_SGI_SN_SIM
 
 config IA64_SGI_SN_XP
         tristate "Support communication between SGI SSIs"
-        depends on MSPEC
+        select IA64_UNCACHED_ALLOCATOR
         help
           An SGI machine can be divided into multiple Single System
           Images which act independently of each other and have
diff --git a/arch/ia64/configs/sn2_defconfig b/arch/ia64/configs/sn2_defconfig
index 6ff7107fee4d..a01bb02d074d 100644
--- a/arch/ia64/configs/sn2_defconfig
+++ b/arch/ia64/configs/sn2_defconfig
@@ -588,6 +588,7 @@ CONFIG_SGI_MBCS=m
 CONFIG_SERIAL_CORE=y
 CONFIG_SERIAL_CORE_CONSOLE=y
 CONFIG_SERIAL_SGI_L1_CONSOLE=y
+CONFIG_SERIAL_SGI_IOC4=y
 CONFIG_UNIX98_PTYS=y
 CONFIG_LEGACY_PTYS=y
 CONFIG_LEGACY_PTY_COUNT=256
@@ -788,6 +789,11 @@ CONFIG_INFINIBAND_IPOIB=m
 # CONFIG_INFINIBAND_IPOIB_DEBUG is not set
 
 #
+# SN Devices
+#
+CONFIG_SGI_IOC4=y
+
+#
 # File systems
 #
 CONFIG_EXT2_FS=y
diff --git a/arch/ia64/defconfig b/arch/ia64/defconfig
index 2aea0f9e6e1d..7be8096e0561 100644
--- a/arch/ia64/defconfig
+++ b/arch/ia64/defconfig
@@ -657,6 +657,7 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y
 CONFIG_SERIAL_CORE=y
 CONFIG_SERIAL_CORE_CONSOLE=y
 CONFIG_SERIAL_SGI_L1_CONSOLE=y
+CONFIG_SERIAL_SGI_IOC4=y
 # CONFIG_SERIAL_JSM is not set
 CONFIG_UNIX98_PTYS=y
 CONFIG_LEGACY_PTYS=y
@@ -976,6 +977,11 @@ CONFIG_INFINIBAND_IPOIB=m
 # CONFIG_INFINIBAND_IPOIB_DEBUG is not set
 
 #
+# SN Devices
+#
+CONFIG_SGI_IOC4=y
+
+#
 # File systems
 #
 CONFIG_EXT2_FS=y
diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
index c1a02bbc252c..4c73d8ba2e3d 100644
--- a/arch/ia64/kernel/Makefile
+++ b/arch/ia64/kernel/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_SMP)		+= smp.o smpboot.o domain.o
 obj-$(CONFIG_PERFMON)           += perfmon_default_smpl.o
 obj-$(CONFIG_IA64_CYCLONE)      += cyclone.o
 obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o
+obj-$(CONFIG_IA64_UNCACHED_ALLOCATOR)   += uncached.o
 mca_recovery-y                  += mca_drv.o mca_drv_asm.o
 
 # The gate DSO image is built using a special linker script.
diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c
index 4a3b1aac43e7..179f230816ed 100644
--- a/arch/ia64/kernel/efi.c
+++ b/arch/ia64/kernel/efi.c
@@ -410,6 +410,38 @@ efi_memmap_walk (efi_freemem_callback_t callback, void *arg)
 }
 
 /*
+ * Walk the EFI memory map to pull out leftover pages in the lower
+ * memory regions which do not end up in the regular memory map and
+ * stick them into the uncached allocator
+ *
+ * The regular walk function is significantly more complex than the
+ * uncached walk which means it really doesn't make sense to try and
+ * marge the two.
+ */
+void __init
+efi_memmap_walk_uc (efi_freemem_callback_t callback)
+{
+        void *efi_map_start, *efi_map_end, *p;
+        efi_memory_desc_t *md;
+        u64 efi_desc_size, start, end;
+
+        efi_map_start = __va(ia64_boot_param->efi_memmap);
+        efi_map_end = efi_map_start + ia64_boot_param->efi_memmap_size;
+        efi_desc_size = ia64_boot_param->efi_memdesc_size;
+
+        for (p = efi_map_start; p < efi_map_end; p += efi_desc_size) {
+                md = p;
+                if (md->attribute == EFI_MEMORY_UC) {
+                        start = PAGE_ALIGN(md->phys_addr);
+                        end = PAGE_ALIGN((md->phys_addr+(md->num_pages << EFI_PAGE_SHIFT)) & PAGE_MASK);
+                        if ((*callback)(start, end, NULL) < 0)
+                                return;
+                }
+        }
+}
+
+
+/*
  * Look for the PAL_CODE region reported by EFI and maps it using an
  * ITR to enable safe PAL calls in virtual mode.  See IA-64 Processor
  * Abstraction Layer chapter 11 in ADAG
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
index d99316c9be28..b1d5d3d5276c 100644
--- a/arch/ia64/kernel/entry.S
+++ b/arch/ia64/kernel/entry.S
@@ -1579,7 +1579,7 @@ sys_call_table:
         data8 sys_keyctl
         data8 sys_ni_syscall
         data8 sys_ni_syscall                    // 1275
-        data8 sys_ni_syscall
+        data8 sys_set_zone_reclaim
         data8 sys_ni_syscall
         data8 sys_ni_syscall
         data8 sys_ni_syscall
diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c
new file mode 100644
index 000000000000..490dfc9ab47f
--- /dev/null
+++ b/arch/ia64/kernel/uncached.c
@@ -0,0 +1,246 @@
+/*
+ * Copyright (C) 2001-2005 Silicon Graphics, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ *
+ * A simple uncached page allocator using the generic allocator. This
+ * allocator first utilizes the spare (spill) pages found in the EFI
+ * memmap and will then start converting cached pages to uncached ones
+ * at a granule at a time. Node awareness is implemented by having a
+ * pool of pages per node.
+ */
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/efi.h>
+#include <linux/genalloc.h>
+#include <asm/page.h>
+#include <asm/pal.h>
+#include <asm/system.h>
+#include <asm/pgtable.h>
+#include <asm/atomic.h>
+#include <asm/tlbflush.h>
+#include <asm/sn/arch.h>
+
+#define DEBUG   0
+
+#if DEBUG
+#define dprintk                 printk
+#else
+#define dprintk(x...)           do { } while (0)
+#endif
+
+void __init efi_memmap_walk_uc (efi_freemem_callback_t callback);
+
+#define MAX_UNCACHED_GRANULES   5
+static int allocated_granules;
+
+struct gen_pool *uncached_pool[MAX_NUMNODES];
+
+
+static void uncached_ipi_visibility(void *data)
+{
+        int status;
+
+        status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
+        if ((status != PAL_VISIBILITY_OK) &&
+            (status != PAL_VISIBILITY_OK_REMOTE_NEEDED))
+                printk(KERN_DEBUG "pal_prefetch_visibility() returns %i on "
+                       "CPU %i\n", status, get_cpu());
+}
+
+
+static void uncached_ipi_mc_drain(void *data)
+{
+        int status;
+        status = ia64_pal_mc_drain();
+        if (status)
+                printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on "
+                       "CPU %i\n", status, get_cpu());
+}
+
+
+static unsigned long
+uncached_get_new_chunk(struct gen_pool *poolp)
+{
+        struct page *page;
+        void *tmp;
+        int status, i;
+        unsigned long addr, node;
+
+        if (allocated_granules >= MAX_UNCACHED_GRANULES)
+                return 0;
+
+        node = poolp->private;
+        page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO,
+                                IA64_GRANULE_SHIFT-PAGE_SHIFT);
+
+        dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n",
+                page, (unsigned long)(page-vmem_map) << PAGE_SHIFT);
+
+        /*
+         * Do magic if no mem on local node! XXX
+         */
+        if (!page)
+                return 0;
+        tmp = page_address(page);
+
+        /*
+         * There's a small race here where it's possible for someone to
+         * access the page through /dev/mem halfway through the conversion
+         * to uncached - not sure it's really worth bothering about
+         */
+        for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
+                SetPageUncached(&page[i]);
+
+        flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE);
+
+        status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
+
+        dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n",
+                status, get_cpu());
+
+        if (!status) {
+                status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1);
+                if (status)
+                        printk(KERN_WARNING "smp_call_function failed for "
+                               "uncached_ipi_visibility! (%i)\n", status);
+        }
+
+        if (ia64_platform_is("sn2"))
+                sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE);
+        else
+                flush_icache_range((unsigned long)tmp,
+                                   (unsigned long)tmp+IA64_GRANULE_SIZE);
+
+        ia64_pal_mc_drain();
+        status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1);
+        if (status)
+                printk(KERN_WARNING "smp_call_function failed for "
+                       "uncached_ipi_mc_drain! (%i)\n", status);
+
+        addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
+
+        allocated_granules++;
+        return addr;
+}
+
+
+/*
+ * uncached_alloc_page
+ *
+ * Allocate 1 uncached page. Allocates on the requested node. If no
+ * uncached pages are available on the requested node, roundrobin starting
+ * with higher nodes.
+ */
+unsigned long
+uncached_alloc_page(int nid)
+{
+        unsigned long maddr;
+
+        maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE);
+
+        dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n",
+                maddr, nid);
+
+        /*
+         * If no memory is availble on our local node, try the
+         * remaining nodes in the system.
+         */
+        if (!maddr) {
+                int i;
+
+                for (i = MAX_NUMNODES - 1; i >= 0; i--) {
+                        if (i == nid || !node_online(i))
+                                continue;
+                        maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE);
+                        dprintk(KERN_DEBUG "uncached_alloc_page alternate search "
+                                "returns %lx on node %i\n", maddr, i);
+                        if (maddr) {
+                                break;
+                        }
+                }
+        }
+
+        return maddr;
+}
+EXPORT_SYMBOL(uncached_alloc_page);
+
+
+/*
+ * uncached_free_page
+ *
+ * Free a single uncached page.
+ */
+void
+uncached_free_page(unsigned long maddr)
+{
+        int node;
+
+        node = nasid_to_cnodeid(NASID_GET(maddr));
+
+        dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node);
+
+        if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
+                panic("uncached_free_page invalid address %lx\n", maddr);
+
+        gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE);
+}
+EXPORT_SYMBOL(uncached_free_page);
+
+
+/*
+ * uncached_build_memmap,
+ *
+ * Called at boot time to build a map of pages that can be used for
+ * memory special operations.
+ */
+static int __init
+uncached_build_memmap(unsigned long start, unsigned long end, void *arg)
+{
+        long length;
+        unsigned long vstart, vend;
+        int node;
+
+        length = end - start;
+        vstart = start + __IA64_UNCACHED_OFFSET;
+        vend = end + __IA64_UNCACHED_OFFSET;
+
+        dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end);
+
+        memset((char *)vstart, 0, length);
+
+        node = nasid_to_cnodeid(NASID_GET(start));
+
+        for (; vstart < vend ; vstart += PAGE_SIZE) {
+                dprintk(KERN_INFO "sticking %lx into the pool!\n", vstart);
+                gen_pool_free(uncached_pool[node], vstart, PAGE_SIZE);
+        }
+
+        return 0;
+}
+
+
+static int __init uncached_init(void) {
+        int i;
+
+        for (i = 0; i < MAX_NUMNODES; i++) {
+                if (!node_online(i))
+                        continue;
+                uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT,
+                                                   &uncached_get_new_chunk, i);
+        }
+
+        efi_memmap_walk_uc(uncached_build_memmap);
+
+        return 0;
+}
+
+__initcall(uncached_init);
diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index df08ae7634b6..e0a776a3044c 100644
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -24,7 +24,7 @@
 
 unsigned int hpage_shift=HPAGE_SHIFT_DEFAULT;
 
-static pte_t *
+pte_t *
 huge_pte_alloc (struct mm_struct *mm, unsigned long addr)
 {
         unsigned long taddr = htlbpage_to_page(addr);
@@ -43,7 +43,7 @@ huge_pte_alloc (struct mm_struct *mm, unsigned long addr)
         return pte;
 }
 
-static pte_t *
+pte_t *
 huge_pte_offset (struct mm_struct *mm, unsigned long addr)
 {
         unsigned long taddr = htlbpage_to_page(addr);
@@ -67,23 +67,6 @@ huge_pte_offset (struct mm_struct *mm, unsigned long addr)
 
 #define mk_pte_huge(entry) { pte_val(entry) |= _PAGE_P; }
 
-static void
-set_huge_pte (struct mm_struct *mm, struct vm_area_struct *vma,
-              struct page *page, pte_t * page_table, int write_access)
-{
-        pte_t entry;
-
-        add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
-        if (write_access) {
-                entry =
-                    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
-        } else
-                entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
-        entry = pte_mkyoung(entry);
-        mk_pte_huge(entry);
-        set_pte(page_table, entry);
-        return;
-}
 /*
  * This function checks for proper alignment of input addr and len parameters.
  */
@@ -99,68 +82,6 @@ int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
         return 0;
 }
 
-int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
-                        struct vm_area_struct *vma)
-{
-        pte_t *src_pte, *dst_pte, entry;
-        struct page *ptepage;
-        unsigned long addr = vma->vm_start;
-        unsigned long end = vma->vm_end;
-
-        while (addr < end) {
-                dst_pte = huge_pte_alloc(dst, addr);
-                if (!dst_pte)
-                        goto nomem;
-                src_pte = huge_pte_offset(src, addr);
-                entry = *src_pte;
-                ptepage = pte_page(entry);
-                get_page(ptepage);
-                set_pte(dst_pte, entry);
-                add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE);
-                addr += HPAGE_SIZE;
-        }
-        return 0;
-nomem:
-        return -ENOMEM;
-}
-
-int
-follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
-                    struct page **pages, struct vm_area_struct **vmas,
-                    unsigned long *st, int *length, int i)
-{
-        pte_t *ptep, pte;
-        unsigned long start = *st;
-        unsigned long pstart;
-        int len = *length;
-        struct page *page;
-
-        do {
-                pstart = start & HPAGE_MASK;
-                ptep = huge_pte_offset(mm, start);
-                pte = *ptep;
-
-back1:
-                page = pte_page(pte);
-                if (pages) {
-                        page += ((start & ~HPAGE_MASK) >> PAGE_SHIFT);
-                        get_page(page);
-                        pages[i] = page;
-                }
-                if (vmas)
-                        vmas[i] = vma;
-                i++;
-                len--;
-                start += PAGE_SIZE;
-                if (((start & HPAGE_MASK) == pstart) && len &&
-                                (start < vma->vm_end))
-                        goto back1;
-        } while (len && start < vma->vm_end);
-        *length = len;
-        *st = start;
-        return i;
-}
-
 struct page *follow_huge_addr(struct mm_struct *mm, unsigned long addr, int write)
 {
         struct page *page;
@@ -212,81 +133,6 @@ void hugetlb_free_pgd_range(struct mmu_gather **tlb,
         free_pgd_range(tlb, addr, end, floor, ceiling);
 }
 
-void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
-{
-        struct mm_struct *mm = vma->vm_mm;
-        unsigned long address;
-        pte_t *pte;
-        struct page *page;
-
-        BUG_ON(start & (HPAGE_SIZE - 1));
-        BUG_ON(end & (HPAGE_SIZE - 1));
-
-        for (address = start; address < end; address += HPAGE_SIZE) {
-                pte = huge_pte_offset(mm, address);
-                if (pte_none(*pte))
-                        continue;
-                page = pte_page(*pte);
-                put_page(page);
-                pte_clear(mm, address, pte);
-        }
-        add_mm_counter(mm, rss, - ((end - start) >> PAGE_SHIFT));
-        flush_tlb_range(vma, start, end);
-}
-
-int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
-{
-        struct mm_struct *mm = current->mm;
-        unsigned long addr;
-        int ret = 0;
-
-        BUG_ON(vma->vm_start & ~HPAGE_MASK);
-        BUG_ON(vma->vm_end & ~HPAGE_MASK);
-
-        spin_lock(&mm->page_table_lock);
-        for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
-                unsigned long idx;
-                pte_t *pte = huge_pte_alloc(mm, addr);
-                struct page *page;
-
-                if (!pte) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                if (!pte_none(*pte))
-                        continue;
-
-                idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)
-                        + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
-                page = find_get_page(mapping, idx);
-                if (!page) {
-                        /* charge the fs quota first */
-                        if (hugetlb_get_quota(mapping)) {
-                                ret = -ENOMEM;
-                                goto out;
-                        }
-                        page = alloc_huge_page();
-                        if (!page) {
-                                hugetlb_put_quota(mapping);
-                                ret = -ENOMEM;
-                                goto out;
-                        }
-                        ret = add_to_page_cache(page, mapping, idx, GFP_ATOMIC);
-                        if (! ret) {
-                                unlock_page(page);
-                        } else {
-                                hugetlb_put_quota(mapping);
-                                page_cache_release(page);
-                                goto out;
-                        }
-                }
-                set_huge_pte(mm, vma, page, pte, vma->vm_flags & VM_WRITE);
-        }
-out:
-        spin_unlock(&mm->page_table_lock);
-        return ret;
-}
-
 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
                 unsigned long pgoff, unsigned long flags)
 {
diff --git a/arch/ia64/sn/kernel/xpc_partition.c b/arch/ia64/sn/kernel/xpc_partition.c
index 2c3c4a8af553..cd7ed73f0e7a 100644
--- a/arch/ia64/sn/kernel/xpc_partition.c
+++ b/arch/ia64/sn/kernel/xpc_partition.c
@@ -22,6 +22,7 @@
 #include <linux/cache.h>
 #include <linux/mmzone.h>
 #include <linux/nodemask.h>
+#include <asm/uncached.h>
 #include <asm/sn/bte.h>
 #include <asm/sn/intr.h>
 #include <asm/sn/sn_sal.h>
@@ -183,7 +184,7 @@ xpc_rsvd_page_init(void)
          * memory protections are never restricted.
          */
         if ((amos_page = xpc_vars->amos_page) == NULL) {
-                amos_page = (AMO_t *) mspec_kalloc_page(0);
+                amos_page = (AMO_t *) TO_AMO(uncached_alloc_page(0));
                 if (amos_page == NULL) {
                         dev_err(xpc_part, "can't allocate page of AMOs\n");
                         return NULL;
@@ -200,7 +201,8 @@ xpc_rsvd_page_init(void)
                         if (ret != 0) {
                                 dev_err(xpc_part, "can't change memory "
                                         "protections\n");
-                                mspec_kfree_page((unsigned long) amos_page);
+                                uncached_free_page(__IA64_UNCACHED_OFFSET |
+                                                   TO_PHYS((u64) amos_page));
                                 return NULL;
                         }
                 }