5 files changed, 123 insertions, 108 deletions

diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
index 292983413ae2..33ca56c90da2 100644
--- a/arch/sparc64/kernel/pci_iommu.c
+++ b/arch/sparc64/kernel/pci_iommu.c
@@ -8,6 +8,7 @@
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
+#include <linux/delay.h>
 
 #include <asm/pbm.h>
 
@@ -379,6 +380,56 @@ bad:
         return PCI_DMA_ERROR_CODE;
 }
 
+static void pci_strbuf_flush(struct pci_strbuf *strbuf, struct pci_iommu *iommu, u32 vaddr, unsigned long ctx, unsigned long npages)
+{
+        int limit;
+
+        PCI_STC_FLUSHFLAG_INIT(strbuf);
+        if (strbuf->strbuf_ctxflush &&
+            iommu->iommu_ctxflush) {
+                unsigned long matchreg, flushreg;
+
+                flushreg = strbuf->strbuf_ctxflush;
+                matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
+
+                limit = 100000;
+                pci_iommu_write(flushreg, ctx);
+                for(;;) {
+                        if (((long)pci_iommu_read(matchreg)) >= 0L)
+                                break;
+                        limit--;
+                        if (!limit)
+                                break;
+                        udelay(1);
+                }
+                if (!limit)
+                        printk(KERN_WARNING "pci_strbuf_flush: ctx flush "
+                               "timeout vaddr[%08x] ctx[%lx]\n",
+                               vaddr, ctx);
+        } else {
+                unsigned long i;
+
+                for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE)
+                        pci_iommu_write(strbuf->strbuf_pflush, vaddr);
+        }
+
+        pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
+        (void) pci_iommu_read(iommu->write_complete_reg);
+
+        limit = 100000;
+        while (!PCI_STC_FLUSHFLAG_SET(strbuf)) {
+                limit--;
+                if (!limit)
+                        break;
+                udelay(1);
+                membar("#LoadLoad");
+        }
+        if (!limit)
+                printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout "
+                       "vaddr[%08x] ctx[%lx] npages[%ld]\n",
+                       vaddr, ctx, npages);
+}
+
 /* Unmap a single streaming mode DMA translation. */
 void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int direction)
 {
@@ -386,7 +437,7 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
         struct pci_iommu *iommu;
         struct pci_strbuf *strbuf;
         iopte_t *base;
-        unsigned long flags, npages, i, ctx;
+        unsigned long flags, npages, ctx;
 
         if (direction == PCI_DMA_NONE)
                 BUG();
@@ -414,29 +465,8 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int
                 ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
 
         /* Step 1: Kick data out of streaming buffers if necessary. */
-        if (strbuf->strbuf_enabled) {
-                u32 vaddr = bus_addr;
-
-                PCI_STC_FLUSHFLAG_INIT(strbuf);
-                if (strbuf->strbuf_ctxflush &&
-                    iommu->iommu_ctxflush) {
-                        unsigned long matchreg, flushreg;
-
-                        flushreg = strbuf->strbuf_ctxflush;
-                        matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
-                        do {
-                                pci_iommu_write(flushreg, ctx);
-                        } while(((long)pci_iommu_read(matchreg)) < 0L);
-                } else {
-                        for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE)
-                                pci_iommu_write(strbuf->strbuf_pflush, vaddr);
-                }
-
-                pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
-                (void) pci_iommu_read(iommu->write_complete_reg);
-                while (!PCI_STC_FLUSHFLAG_SET(strbuf))
-                        membar("#LoadLoad");
-        }
+        if (strbuf->strbuf_enabled)
+                pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
 
         /* Step 2: Clear out first TSB entry. */
         iopte_make_dummy(iommu, base);
@@ -647,29 +677,8 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems,
                 ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
 
         /* Step 1: Kick data out of streaming buffers if necessary. */
-        if (strbuf->strbuf_enabled) {
-                u32 vaddr = (u32) bus_addr;
-
-                PCI_STC_FLUSHFLAG_INIT(strbuf);
-                if (strbuf->strbuf_ctxflush &&
-                    iommu->iommu_ctxflush) {
-                        unsigned long matchreg, flushreg;
-
-                        flushreg = strbuf->strbuf_ctxflush;
-                        matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
-                        do {
-                                pci_iommu_write(flushreg, ctx);
-                        } while(((long)pci_iommu_read(matchreg)) < 0L);
-                } else {
-                        for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE)
-                                pci_iommu_write(strbuf->strbuf_pflush, vaddr);
-                }
-
-                pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
-                (void) pci_iommu_read(iommu->write_complete_reg);
-                while (!PCI_STC_FLUSHFLAG_SET(strbuf))
-                        membar("#LoadLoad");
-        }
+        if (strbuf->strbuf_enabled)
+                pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
 
         /* Step 2: Clear out first TSB entry. */
         iopte_make_dummy(iommu, base);
@@ -715,28 +724,7 @@ void pci_dma_sync_single_for_cpu(struct pci_dev *pdev, dma_addr_t bus_addr, size
         }
 
         /* Step 2: Kick data out of streaming buffers. */
-        PCI_STC_FLUSHFLAG_INIT(strbuf);
-        if (iommu->iommu_ctxflush &&
-            strbuf->strbuf_ctxflush) {
-                unsigned long matchreg, flushreg;
-
-                flushreg = strbuf->strbuf_ctxflush;
-                matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
-                do {
-                        pci_iommu_write(flushreg, ctx);
-                } while(((long)pci_iommu_read(matchreg)) < 0L);
-        } else {
-                unsigned long i;
-
-                for (i = 0; i < npages; i++, bus_addr += IO_PAGE_SIZE)
-                        pci_iommu_write(strbuf->strbuf_pflush, bus_addr);
-        }
-
-        /* Step 3: Perform flush synchronization sequence. */
-        pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
-        (void) pci_iommu_read(iommu->write_complete_reg);
-        while (!PCI_STC_FLUSHFLAG_SET(strbuf))
-                membar("#LoadLoad");
+        pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
 
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
@@ -749,7 +737,8 @@ void pci_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, i
         struct pcidev_cookie *pcp;
         struct pci_iommu *iommu;
         struct pci_strbuf *strbuf;
-        unsigned long flags, ctx;
+        unsigned long flags, ctx, npages, i;
+        u32 bus_addr;
 
         pcp = pdev->sysdata;
         iommu = pcp->pbm->iommu;
@@ -772,36 +761,14 @@ void pci_dma_sync_sg_for_cpu(struct pci_dev *pdev, struct scatterlist *sglist, i
         }
 
         /* Step 2: Kick data out of streaming buffers. */
-        PCI_STC_FLUSHFLAG_INIT(strbuf);
-        if (iommu->iommu_ctxflush &&
-            strbuf->strbuf_ctxflush) {
-                unsigned long matchreg, flushreg;
-
-                flushreg = strbuf->strbuf_ctxflush;
-                matchreg = PCI_STC_CTXMATCH_ADDR(strbuf, ctx);
-                do {
-                        pci_iommu_write(flushreg, ctx);
-                } while (((long)pci_iommu_read(matchreg)) < 0L);
-        } else {
-                unsigned long i, npages;
-                u32 bus_addr;
-
-                bus_addr = sglist[0].dma_address & IO_PAGE_MASK;
-
-                for(i = 1; i < nelems; i++)
-                        if (!sglist[i].dma_length)
-                                break;
-                i--;
-                npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - bus_addr) >> IO_PAGE_SHIFT;
-                for (i = 0; i < npages; i++, bus_addr += IO_PAGE_SIZE)
-                        pci_iommu_write(strbuf->strbuf_pflush, bus_addr);
-        }
-
-        /* Step 3: Perform flush synchronization sequence. */
-        pci_iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa);
-        (void) pci_iommu_read(iommu->write_complete_reg);
-        while (!PCI_STC_FLUSHFLAG_SET(strbuf))
-                membar("#LoadLoad");
+        bus_addr = sglist[0].dma_address & IO_PAGE_MASK;
+        for(i = 1; i < nelems; i++)
+                if (!sglist[i].dma_length)
+                        break;
+        i--;
+        npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length)
+                  - bus_addr) >> IO_PAGE_SHIFT;
+        pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages);
 
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
diff --git a/arch/sparc64/kernel/sbus.c b/arch/sparc64/kernel/sbus.c
index 14d9c3a21b9a..76ea6455433f 100644
--- a/arch/sparc64/kernel/sbus.c
+++ b/arch/sparc64/kernel/sbus.c
@@ -117,19 +117,34 @@ static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages
 
 #define STRBUF_TAG_VALID        0x02UL
 
-static void strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
+static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages)
 {
+        unsigned long n;
+        int limit;
+
         iommu->strbuf_flushflag = 0UL;
-        while (npages--)
-                upa_writeq(base + (npages << IO_PAGE_SHIFT),
+        n = npages;
+        while (n--)
+                upa_writeq(base + (n << IO_PAGE_SHIFT),
                            iommu->strbuf_regs + STRBUF_PFLUSH);
 
         /* Whoopee cushion! */
         upa_writeq(__pa(&iommu->strbuf_flushflag),
                    iommu->strbuf_regs + STRBUF_FSYNC);
         upa_readq(iommu->sbus_control_reg);
-        while (iommu->strbuf_flushflag == 0UL)
+
+        limit = 100000;
+        while (iommu->strbuf_flushflag == 0UL) {
+                limit--;
+                if (!limit)
+                        break;
+                udelay(1);
                 membar("#LoadLoad");
+        }
+        if (!limit)
+                printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout "
+                       "vaddr[%08x] npages[%ld]\n",
+                       base, npages);
 }
 
 static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages)
@@ -406,7 +421,7 @@ void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size,
 
         spin_lock_irqsave(&iommu->lock, flags);
         free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT);
-        strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT);
+        sbus_strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT);
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
@@ -569,7 +584,7 @@ void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int
         iommu = sdev->bus->iommu;
         spin_lock_irqsave(&iommu->lock, flags);
         free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT);
-        strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT);
+        sbus_strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT);
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
@@ -581,7 +596,7 @@ void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t base, size_t
         size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK));
 
         spin_lock_irqsave(&iommu->lock, flags);
-        strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT);
+        sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT);
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
@@ -605,7 +620,7 @@ void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sg, int
         size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base;
 
         spin_lock_irqsave(&iommu->lock, flags);
-        strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT);
+        sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT);
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
diff --git a/arch/sparc64/kernel/setup.c b/arch/sparc64/kernel/setup.c
index 12c3d84b7460..b7e6a91952b2 100644
--- a/arch/sparc64/kernel/setup.c
+++ b/arch/sparc64/kernel/setup.c
@@ -383,6 +383,17 @@ static void __init process_switch(char c)
                 /* Use PROM debug console. */
                 register_console(&prom_debug_console);
                 break;
+        case 'P':
+                /* Force UltraSPARC-III P-Cache on. */
+                if (tlb_type != cheetah) {
+                        printk("BOOT: Ignoring P-Cache force option.\n");
+                        break;
+                }
+                cheetah_pcache_forced_on = 1;
+                add_taint(TAINT_MACHINE_CHECK);
+                cheetah_enable_pcache();
+                break;
+
         default:
                 printk("Unknown boot switch (-%c)\n", c);
                 break;
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 6dff06a44e76..e5b9c7a27789 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -123,6 +123,9 @@ void __init smp_callin(void)
 
         smp_setup_percpu_timer();
 
+        if (cheetah_pcache_forced_on)
+                cheetah_enable_pcache();
+
         local_irq_enable();
 
         calibrate_delay();
diff --git a/arch/sparc64/kernel/traps.c b/arch/sparc64/kernel/traps.c
index 56b203a2af69..a9f4596d7c2b 100644
--- a/arch/sparc64/kernel/traps.c
+++ b/arch/sparc64/kernel/traps.c
@@ -421,6 +421,25 @@ asmlinkage void cee_log(unsigned long ce_status,
         }
 }
 
+int cheetah_pcache_forced_on;
+
+void cheetah_enable_pcache(void)
+{
+        unsigned long dcr;
+
+        printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
+               smp_processor_id());
+
+        __asm__ __volatile__("ldxa [%%g0] %1, %0"
+                             : "=r" (dcr)
+                             : "i" (ASI_DCU_CONTROL_REG));
+        dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
+        __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
+                             "membar #Sync"
+                             : /* no outputs */
+                             : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
+}
+
 /* Cheetah error trap handling. */
 static unsigned long ecache_flush_physbase;
 static unsigned long ecache_flush_linesize;