[POWERPC] U4 DART improvements

Better late than never...

Respin based on previous comment. Only remaining issue last time was an
extra mb() that I've taken out.

Signed-off-by: Paul Mackerras <paulus@samba.org>

2 files changed, 48 insertions, 7 deletions

diff --git a/arch/powerpc/sysdev/dart.h b/arch/powerpc/sysdev/dart.h
index c2d05763ccbe..1c8817c4835e 100644
--- a/arch/powerpc/sysdev/dart.h
+++ b/arch/powerpc/sysdev/dart.h
@@ -47,8 +47,12 @@
 /* U4 registers */
 #define DART_BASE_U4_BASE_MASK  0xffffff
 #define DART_BASE_U4_BASE_SHIFT 0
-#define DART_CNTL_U4_FLUSHTLB   0x20000000
 #define DART_CNTL_U4_ENABLE     0x80000000
+#define DART_CNTL_U4_IONE       0x40000000
+#define DART_CNTL_U4_FLUSHTLB   0x20000000
+#define DART_CNTL_U4_IDLE       0x10000000
+#define DART_CNTL_U4_PAR_EN     0x08000000
+#define DART_CNTL_U4_IONE_MASK  0x07ffffff
 #define DART_SIZE_U4_SIZE_MASK  0x1fff
 #define DART_SIZE_U4_SIZE_SHIFT 0
 
diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c
index 6232091cc72b..7c7f34ce4986 100644
--- a/arch/powerpc/sysdev/dart_iommu.c
+++ b/arch/powerpc/sysdev/dart_iommu.c
@@ -101,8 +101,8 @@ retry:
         if (l == (1L << limit)) {
                 if (limit < 4) {
                         limit++;
-                        reg = DART_IN(DART_CNTL);
-                        reg &= ~inv_bit;
+                        reg = DART_IN(DART_CNTL);
+                        reg &= ~inv_bit;
                         DART_OUT(DART_CNTL, reg);
                         goto retry;
                 } else
@@ -111,11 +111,39 @@ retry:
         }
 }
 
+static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
+{
+        unsigned int reg;
+        unsigned int l, limit;
+
+        reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
+                (bus_rpn & DART_CNTL_U4_IONE_MASK);
+        DART_OUT(DART_CNTL, reg);
+
+        limit = 0;
+wait_more:
+        l = 0;
+        while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
+                rmb();
+                l++;
+        }
+
+        if (l == (1L << limit)) {
+                if (limit < 4) {
+                        limit++;
+                        goto wait_more;
+                } else
+                        panic("DART: TLB did not flush after waiting a long "
+                              "time. Buggy U4 ?");
+        }
+}
+
 static void dart_flush(struct iommu_table *tbl)
 {
-        if (dart_dirty)
+        if (dart_dirty) {
                 dart_tlb_invalidate_all();
-        dart_dirty = 0;
+                dart_dirty = 0;
+        }
 }
 
 static void dart_build(struct iommu_table *tbl, long index,
@@ -124,6 +152,7 @@ static void dart_build(struct iommu_table *tbl, long index,
 {
         unsigned int *dp;
         unsigned int rpn;
+        long l;
 
         DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
 
@@ -135,7 +164,8 @@ static void dart_build(struct iommu_table *tbl, long index,
         /* On U3, all memory is contigous, so we can move this
          * out of the loop.
          */
-        while (npages--) {
+        l = npages;
+        while (l--) {
                 rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT;
 
                 *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
@@ -143,7 +173,14 @@ static void dart_build(struct iommu_table *tbl, long index,
                 uaddr += DART_PAGE_SIZE;
         }
 
-        dart_dirty = 1;
+        if (dart_is_u4) {
+                rpn = index;
+                mb(); /* make sure all updates have reached memory */
+                while (npages--)
+                        dart_tlb_invalidate_one(rpn++);
+        } else {
+                dart_dirty = 1;
+        }
 }