parisc: Use per-pagetable spinlock

PA-RISC uses a global spinlock to protect pagetable updates in the TLB
fault handlers. When multiple cores are taking TLB faults simultaneously,
the cache line containing the spinlock becomes a bottleneck.

This patch embeds the spinlock in the top level page directory, so that
every process has its own lock. It improves performance by 30% when
doing parallel compilations.

At least on the N class systems, only one PxTLB inter processor
broadcast can be active at any one time on the Merced bus. If a Merced
bus is found, this patch serializes the TLB flushes with the
pa_tlb_flush_lock spinlock.

v1: Initial patch by Mikulas
v2: Added Merced detection by Helge
v3: Revised TLB serialization by Dave & Helge

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: John David Anglin <dave.anglin@bell.net>
Signed-off-by: Helge Deller <deller@gmx.de>

9 files changed, 108 insertions, 49 deletions

diff --git a/arch/parisc/include/asm/hardware.h b/arch/parisc/include/asm/hardware.h
index d6e1ed145031..9d3d7737c58b 100644
--- a/arch/parisc/include/asm/hardware.h
+++ b/arch/parisc/include/asm/hardware.h
@@ -120,7 +120,7 @@ extern void get_pci_node_path(struct pci_dev *dev, struct hardware_path *path);
 extern void init_parisc_bus(void);
 extern struct device *hwpath_to_device(struct hardware_path *modpath);
 extern void device_to_hwpath(struct device *dev, struct hardware_path *path);
-
+extern int machine_has_merced_bus(void);
 
 /* inventory.c: */
 extern void do_memory_inventory(void);
diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h
index d05c678c77c4..ea75cc966dae 100644
--- a/arch/parisc/include/asm/pgalloc.h
+++ b/arch/parisc/include/asm/pgalloc.h
@@ -41,6 +41,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
                 __pgd_val_set(*pgd, PxD_FLAG_ATTACHED);
 #endif
         }
+        spin_lock_init(pgd_spinlock(actual_pgd));
         return actual_pgd;
 }
 
diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h
index c7bb74e22436..a39b079e73f2 100644
--- a/arch/parisc/include/asm/pgtable.h
+++ b/arch/parisc/include/asm/pgtable.h
@@ -17,7 +17,7 @@
 #include <asm/processor.h>
 #include <asm/cache.h>
 
-extern spinlock_t pa_tlb_lock;
+static inline spinlock_t *pgd_spinlock(pgd_t *);
 
 /*
  * kern_addr_valid(ADDR) tests if ADDR is pointing to valid kernel
@@ -34,16 +34,46 @@ extern spinlock_t pa_tlb_lock;
  */
 #define kern_addr_valid(addr)   (1)
 
-/* Purge data and instruction TLB entries.  Must be called holding
- * the pa_tlb_lock.  The TLB purge instructions are slow on SMP
- * machines since the purge must be broadcast to all CPUs.
+/* This is for the serialization of PxTLB broadcasts. At least on the N class
+ * systems, only one PxTLB inter processor broadcast can be active at any one
+ * time on the Merced bus.
+
+ * PTE updates are protected by locks in the PMD.
+ */
+extern spinlock_t pa_tlb_flush_lock;
+extern spinlock_t pa_swapper_pg_lock;
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+extern int pa_serialize_tlb_flushes;
+#else
+#define pa_serialize_tlb_flushes        (0)
+#endif
+
+#define purge_tlb_start(flags)  do { \
+        if (pa_serialize_tlb_flushes)   \
+                spin_lock_irqsave(&pa_tlb_flush_lock, flags); \
+        else \
+                local_irq_save(flags);  \
+        } while (0)
+#define purge_tlb_end(flags)    do { \
+        if (pa_serialize_tlb_flushes)   \
+                spin_unlock_irqrestore(&pa_tlb_flush_lock, flags); \
+        else \
+                local_irq_restore(flags); \
+        } while (0)
+
+/* Purge data and instruction TLB entries. The TLB purge instructions
+ * are slow on SMP machines since the purge must be broadcast to all CPUs.
  */
 
 static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
 {
+        unsigned long flags;
+
+        purge_tlb_start(flags);
         mtsp(mm->context, 1);
         pdtlb(addr);
         pitlb(addr);
+        purge_tlb_end(flags);
 }
 
 /* Certain architectures need to do special things when PTEs
@@ -59,11 +89,11 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
         do {                                                    \
                 pte_t old_pte;                                  \
                 unsigned long flags;                            \
-                spin_lock_irqsave(&pa_tlb_lock, flags);         \
+                spin_lock_irqsave(pgd_spinlock((mm)->pgd), flags);\
                 old_pte = *ptep;                                \
                 set_pte(ptep, pteval);                          \
                 purge_tlb_entries(mm, addr);                    \
-                spin_unlock_irqrestore(&pa_tlb_lock, flags);    \
+                spin_unlock_irqrestore(pgd_spinlock((mm)->pgd), flags);\
         } while (0)
 
 #endif /* !__ASSEMBLY__ */
@@ -88,10 +118,10 @@ static inline void purge_tlb_entries(struct mm_struct *mm, unsigned long addr)
 #if CONFIG_PGTABLE_LEVELS == 3
 #define PGD_ORDER       1 /* Number of pages per pgd */
 #define PMD_ORDER       1 /* Number of pages per pmd */
-#define PGD_ALLOC_ORDER 2 /* first pgd contains pmd */
+#define PGD_ALLOC_ORDER (2 + 1) /* first pgd contains pmd */
 #else
 #define PGD_ORDER       1 /* Number of pages per pgd */
-#define PGD_ALLOC_ORDER PGD_ORDER
+#define PGD_ALLOC_ORDER (PGD_ORDER + 1)
 #endif
 
 /* Definitions for 3rd level (we use PLD here for Page Lower directory
@@ -459,6 +489,15 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *);
 #define __pte_to_swp_entry(pte)         ((swp_entry_t) { pte_val(pte) })
 #define __swp_entry_to_pte(x)           ((pte_t) { (x).val })
 
+
+static inline spinlock_t *pgd_spinlock(pgd_t *pgd)
+{
+        if (unlikely(pgd == swapper_pg_dir))
+                return &pa_swapper_pg_lock;
+        return (spinlock_t *)((char *)pgd + (PAGE_SIZE << (PGD_ALLOC_ORDER - 1)));
+}
+
+
 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
 {
         pte_t pte;
@@ -467,15 +506,15 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned
         if (!pte_young(*ptep))
                 return 0;
 
-        spin_lock_irqsave(&pa_tlb_lock, flags);
+        spin_lock_irqsave(pgd_spinlock(vma->vm_mm->pgd), flags);
         pte = *ptep;
         if (!pte_young(pte)) {
-                spin_unlock_irqrestore(&pa_tlb_lock, flags);
+                spin_unlock_irqrestore(pgd_spinlock(vma->vm_mm->pgd), flags);
                 return 0;
         }
         set_pte(ptep, pte_mkold(pte));
         purge_tlb_entries(vma->vm_mm, addr);
-        spin_unlock_irqrestore(&pa_tlb_lock, flags);
+        spin_unlock_irqrestore(pgd_spinlock(vma->vm_mm->pgd), flags);
         return 1;
 }
 
@@ -485,11 +524,11 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
         pte_t old_pte;
         unsigned long flags;
 
-        spin_lock_irqsave(&pa_tlb_lock, flags);
+        spin_lock_irqsave(pgd_spinlock(mm->pgd), flags);
         old_pte = *ptep;
         set_pte(ptep, __pte(0));
         purge_tlb_entries(mm, addr);
-        spin_unlock_irqrestore(&pa_tlb_lock, flags);
+        spin_unlock_irqrestore(pgd_spinlock(mm->pgd), flags);
 
         return old_pte;
 }
@@ -497,10 +536,10 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
         unsigned long flags;
-        spin_lock_irqsave(&pa_tlb_lock, flags);
+        spin_lock_irqsave(pgd_spinlock(mm->pgd), flags);
         set_pte(ptep, pte_wrprotect(*ptep));
         purge_tlb_entries(mm, addr);
-        spin_unlock_irqrestore(&pa_tlb_lock, flags);
+        spin_unlock_irqrestore(pgd_spinlock(mm->pgd), flags);
 }
 
 #define pte_same(A,B)   (pte_val(A) == pte_val(B))
diff --git a/arch/parisc/include/asm/tlbflush.h b/arch/parisc/include/asm/tlbflush.h
index 6804374efa66..c5ded01d45be 100644
--- a/arch/parisc/include/asm/tlbflush.h
+++ b/arch/parisc/include/asm/tlbflush.h
@@ -8,21 +8,6 @@
 #include <linux/sched.h>
 #include <asm/mmu_context.h>
 
-
-/* This is for the serialisation of PxTLB broadcasts.  At least on the
- * N class systems, only one PxTLB inter processor broadcast can be
- * active at any one time on the Merced bus.  This tlb purge
- * synchronisation is fairly lightweight and harmless so we activate
- * it on all systems not just the N class.
-
- * It is also used to ensure PTE updates are atomic and consistent
- * with the TLB.
- */
-extern spinlock_t pa_tlb_lock;
-
-#define purge_tlb_start(flags)  spin_lock_irqsave(&pa_tlb_lock, flags)
-#define purge_tlb_end(flags)    spin_unlock_irqrestore(&pa_tlb_lock, flags)
-
 extern void flush_tlb_all(void);
 extern void flush_tlb_all_local(void *);
 
@@ -79,13 +64,6 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
 static inline void flush_tlb_page(struct vm_area_struct *vma,
         unsigned long addr)
 {
-        unsigned long flags, sid;
-
-        sid = vma->vm_mm->context;
-        purge_tlb_start(flags);
-        mtsp(sid, 1);
-        pdtlb(addr);
-        pitlb(addr);
-        purge_tlb_end(flags);
+        purge_tlb_entries(vma->vm_mm, addr);
 }
 #endif
diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 804880efa11e..0338561968a4 100644
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -40,12 +40,19 @@ void purge_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr);
 void flush_icache_page_asm(unsigned long phys_addr, unsigned long vaddr);
 
 
-/* On some machines (e.g. ones with the Merced bus), there can be
+/* On some machines (i.e., ones with the Merced bus), there can be
  * only a single PxTLB broadcast at a time; this must be guaranteed
- * by software.  We put a spinlock around all TLB flushes  to
- * ensure this.
+ * by software. We need a spinlock around all TLB flushes to ensure
+ * this.
  */
-DEFINE_SPINLOCK(pa_tlb_lock);
+DEFINE_SPINLOCK(pa_tlb_flush_lock);
+
+/* Swapper page setup lock. */
+DEFINE_SPINLOCK(pa_swapper_pg_lock);
+
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+int pa_serialize_tlb_flushes __read_mostly;
+#endif
 
 struct pdc_cache_info cache_info __read_mostly;
 #ifndef CONFIG_PA20
diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c
index 5eb979d04b90..15e7b3be7b6b 100644
--- a/arch/parisc/kernel/drivers.c
+++ b/arch/parisc/kernel/drivers.c
@@ -38,6 +38,7 @@
 #include <asm/io.h>
 #include <asm/pdc.h>
 #include <asm/parisc-device.h>
+#include <asm/ropes.h>
 
 /* See comments in include/asm-parisc/pci.h */
 const struct dma_map_ops *hppa_dma_ops __read_mostly;
@@ -257,6 +258,30 @@ static struct parisc_device *find_device_by_addr(unsigned long hpa)
         return ret ? d.dev : NULL;
 }
 
+static int __init is_IKE_device(struct device *dev, void *data)
+{
+        struct parisc_device *pdev = to_parisc_device(dev);
+
+        if (!check_dev(dev))
+                return 0;
+        if (pdev->id.hw_type != HPHW_BCPORT)
+                return 0;
+        if (IS_IKE(pdev) ||
+                (pdev->id.hversion == REO_MERCED_PORT) ||
+                (pdev->id.hversion == REOG_MERCED_PORT)) {
+                        return 1;
+        }
+        return 0;
+}
+
+int __init machine_has_merced_bus(void)
+{
+        int ret;
+
+        ret = for_each_padev(is_IKE_device, NULL);
+        return ret ? 1 : 0;
+}
+
 /**
  * find_pa_parent_type - Find a parent of a specific type
  * @dev: The device to start searching from
diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S
index 5796524a3137..a1fc04570ade 100644
--- a/arch/parisc/kernel/entry.S
+++ b/arch/parisc/kernel/entry.S
@@ -50,12 +50,8 @@
 
         .import         pa_tlb_lock,data
         .macro  load_pa_tlb_lock reg
-#if __PA_LDCW_ALIGNMENT > 4
-        load32  PA(pa_tlb_lock) + __PA_LDCW_ALIGNMENT-1, \reg
-        depi    0,31,__PA_LDCW_ALIGN_ORDER, \reg
-#else
-        load32  PA(pa_tlb_lock), \reg
-#endif
+        mfctl           %cr25,\reg
+        addil           L%(PAGE_SIZE << (PGD_ALLOC_ORDER - 1)),\reg
         .endm
 
         /* space_to_prot macro creates a prot id from a space id */
diff --git a/arch/parisc/kernel/inventory.c b/arch/parisc/kernel/inventory.c
index 35d05fdd7483..6f2d611347a1 100644
--- a/arch/parisc/kernel/inventory.c
+++ b/arch/parisc/kernel/inventory.c
@@ -31,6 +31,7 @@
 #include <asm/processor.h>
 #include <asm/page.h>
 #include <asm/parisc-device.h>
+#include <asm/tlbflush.h>
 
 /*
 ** Debug options
@@ -638,4 +639,10 @@ void __init do_device_inventory(void)
         }
         printk(KERN_INFO "Found devices:\n");
         print_parisc_devices();
+
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+        pa_serialize_tlb_flushes = machine_has_merced_bus();
+        if (pa_serialize_tlb_flushes)
+                pr_info("Merced bus found: Enable PxTLB serialization.\n");
+#endif
 }
diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c
index d908058d05c1..e05cb2a5c16d 100644
--- a/arch/parisc/kernel/setup.c
+++ b/arch/parisc/kernel/setup.c
@@ -343,6 +343,12 @@ static int __init parisc_init(void)
                         boot_cpu_data.cpu_hz / 1000000,
                         boot_cpu_data.cpu_hz % 1000000  );
 
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+        /* Don't serialize TLB flushes if we run on one CPU only. */
+        if (num_online_cpus() == 1)
+                pa_serialize_tlb_flushes = 0;
+#endif
+
         apply_alternatives_all();
         parisc_setup_cache_timing();