[POWERPC] Allow hooking of PCI MMIO & PIO accessors on 64 bits

This patch reworks the way iSeries hooks on PCI IO operations (both MMIO
and PIO) and provides a generic way for other platforms to do so (we
have need to do that for various other platforms).

While reworking the IO ops, I ended up doing some spring cleaning in
io.h and eeh.h which I might want to split into 2 or 3 patches (among
others, eeh.h had a lot of useless stuff in it).

A side effect is that EEH for PIO should work now (it used to pass IO
ports down to the eeh address check functions which is bogus).

Also, new are MMIO "repeat" ops, which other archs like ARM already had,
and that we have too now: readsb, readsw, readsl, writesb, writesw,
writesl.

In the long run, I might also make EEH use the hooks instead
of wrapping at the toplevel, which would make things even cleaner and
relegate EEH completely in platforms/iseries, but we have to measure the
performance impact there (though it's really only on MMIO reads)

Since I also need to hook on ioremap, I shuffled the functions a bit
there. I introduced ioremap_flags() to use by drivers who want to pass
explicit flags to ioremap (and it can be hooked). The old __ioremap() is
still there as a low level and cannot be hooked, thus drivers who use it
should migrate unless they know they want the low level version.

The patch "arch provides generic iomap missing accessors" (should be
number 4 in this series) is a pre-requisite to provide full iomap
API support with this patch.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Paul Mackerras <paulus@samba.org>

1 files changed, 155 insertions, 215 deletions

diff --git a/arch/powerpc/platforms/iseries/pci.c b/arch/powerpc/platforms/iseries/pci.c
index a90ae42a7bc2..4a06d9c34986 100644
--- a/arch/powerpc/platforms/iseries/pci.c
+++ b/arch/powerpc/platforms/iseries/pci.c
@@ -156,53 +156,6 @@ static void pci_Log_Error(char *Error_Text, int Bus, int SubBus,
 }
 
 /*
- * iSeries_pcibios_init
- *
- * Description:
- *   This function checks for all possible system PCI host bridges that connect
- *   PCI buses.  The system hypervisor is queried as to the guest partition
- *   ownership status.  A pci_controller is built for any bus which is partially
- *   owned or fully owned by this guest partition.
- */
-void iSeries_pcibios_init(void)
-{
-        struct pci_controller *phb;
-        struct device_node *root = of_find_node_by_path("/");
-        struct device_node *node = NULL;
-
-        if (root == NULL) {
-                printk(KERN_CRIT "iSeries_pcibios_init: can't find root "
-                                "of device tree\n");
-                return;
-        }
-        while ((node = of_get_next_child(root, node)) != NULL) {
-                HvBusNumber bus;
-                const u32 *busp;
-
-                if ((node->type == NULL) || (strcmp(node->type, "pci") != 0))
-                        continue;
-
-                busp = get_property(node, "bus-range", NULL);
-                if (busp == NULL)
-                        continue;
-                bus = *busp;
-                printk("bus %d appears to exist\n", bus);
-                phb = pcibios_alloc_controller(node);
-                if (phb == NULL)
-                        continue;
-
-                phb->pci_mem_offset = phb->local_number = bus;
-                phb->first_busno = bus;
-                phb->last_busno = bus;
-                phb->ops = &iSeries_pci_ops;
-        }
-
-        of_node_put(root);
-
-        pci_devs_phb_init();
-}
-
-/*
  * iSeries_pci_final_fixup(void)
  */
 void __init iSeries_pci_final_fixup(void)
@@ -438,11 +391,7 @@ static inline struct device_node *xlate_iomm_address(
 /*
  * Read MM I/O Instructions for the iSeries
  * On MM I/O error, all ones are returned and iSeries_pci_IoError is cal
- * else, data is returned in big Endian format.
- *
- * iSeries_Read_Byte = Read Byte  ( 8 bit)
- * iSeries_Read_Word = Read Word  (16 bit)
- * iSeries_Read_Long = Read Long  (32 bit)
+ * else, data is returned in Big Endian format.
  */
 static u8 iSeries_Read_Byte(const volatile void __iomem *IoAddress)
 {
@@ -462,14 +411,15 @@ static u8 iSeries_Read_Byte(const volatile void __iomem *IoAddress)
                         num_printed = 0;
                 }
                 if (num_printed++ < 10)
-                        printk(KERN_ERR "iSeries_Read_Byte: invalid access at IO address %p\n", IoAddress);
+                        printk(KERN_ERR "iSeries_Read_Byte: invalid access at IO address %p\n",
+                               IoAddress);
                 return 0xff;
         }
         do {
                 HvCall3Ret16(HvCallPciBarLoad8, &ret, dsa, BarOffset, 0);
         } while (CheckReturnCode("RDB", DevNode, &retry, ret.rc) != 0);
 
-        return (u8)ret.value;
+        return ret.value;
 }
 
 static u16 iSeries_Read_Word(const volatile void __iomem *IoAddress)
@@ -490,7 +440,8 @@ static u16 iSeries_Read_Word(const volatile void __iomem *IoAddress)
                         num_printed = 0;
                 }
                 if (num_printed++ < 10)
-                        printk(KERN_ERR "iSeries_Read_Word: invalid access at IO address %p\n", IoAddress);
+                        printk(KERN_ERR "iSeries_Read_Word: invalid access at IO address %p\n",
+                               IoAddress);
                 return 0xffff;
         }
         do {
@@ -498,7 +449,7 @@ static u16 iSeries_Read_Word(const volatile void __iomem *IoAddress)
                                 BarOffset, 0);
         } while (CheckReturnCode("RDW", DevNode, &retry, ret.rc) != 0);
 
-        return swab16((u16)ret.value);
+        return ret.value;
 }
 
 static u32 iSeries_Read_Long(const volatile void __iomem *IoAddress)
@@ -519,7 +470,8 @@ static u32 iSeries_Read_Long(const volatile void __iomem *IoAddress)
                         num_printed = 0;
                 }
                 if (num_printed++ < 10)
-                        printk(KERN_ERR "iSeries_Read_Long: invalid access at IO address %p\n", IoAddress);
+                        printk(KERN_ERR "iSeries_Read_Long: invalid access at IO address %p\n",
+                               IoAddress);
                 return 0xffffffff;
         }
         do {
@@ -527,15 +479,12 @@ static u32 iSeries_Read_Long(const volatile void __iomem *IoAddress)
                                 BarOffset, 0);
         } while (CheckReturnCode("RDL", DevNode, &retry, ret.rc) != 0);
 
-        return swab32((u32)ret.value);
+        return ret.value;
 }
 
 /*
  * Write MM I/O Instructions for the iSeries
  *
- * iSeries_Write_Byte = Write Byte (8 bit)
- * iSeries_Write_Word = Write Word(16 bit)
- * iSeries_Write_Long = Write Long(32 bit)
  */
 static void iSeries_Write_Byte(u8 data, volatile void __iomem *IoAddress)
 {
@@ -581,11 +530,12 @@ static void iSeries_Write_Word(u16 data, volatile void __iomem *IoAddress)
                         num_printed = 0;
                 }
                 if (num_printed++ < 10)
-                        printk(KERN_ERR "iSeries_Write_Word: invalid access at IO address %p\n", IoAddress);
+                        printk(KERN_ERR "iSeries_Write_Word: invalid access at IO address %p\n",
+                               IoAddress);
                 return;
         }
         do {
-                rc = HvCall4(HvCallPciBarStore16, dsa, BarOffset, swab16(data), 0);
+                rc = HvCall4(HvCallPciBarStore16, dsa, BarOffset, data, 0);
         } while (CheckReturnCode("WWW", DevNode, &retry, rc) != 0);
 }
 
@@ -607,231 +557,221 @@ static void iSeries_Write_Long(u32 data, volatile void __iomem *IoAddress)
                         num_printed = 0;
                 }
                 if (num_printed++ < 10)
-                        printk(KERN_ERR "iSeries_Write_Long: invalid access at IO address %p\n", IoAddress);
+                        printk(KERN_ERR "iSeries_Write_Long: invalid access at IO address %p\n",
+                               IoAddress);
                 return;
         }
         do {
-                rc = HvCall4(HvCallPciBarStore32, dsa, BarOffset, swab32(data), 0);
+                rc = HvCall4(HvCallPciBarStore32, dsa, BarOffset, data, 0);
         } while (CheckReturnCode("WWL", DevNode, &retry, rc) != 0);
 }
 
-extern unsigned char __raw_readb(const volatile void __iomem *addr)
+static u8 iseries_readb(const volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        return *(volatile unsigned char __force *)addr;
+        return iSeries_Read_Byte(addr);
 }
-EXPORT_SYMBOL(__raw_readb);
 
-extern unsigned short __raw_readw(const volatile void __iomem *addr)
+static u16 iseries_readw(const volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        return *(volatile unsigned short __force *)addr;
+        return le16_to_cpu(iSeries_Read_Word(addr));
 }
-EXPORT_SYMBOL(__raw_readw);
 
-extern unsigned int __raw_readl(const volatile void __iomem *addr)
+static u32 iseries_readl(const volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        return *(volatile unsigned int __force *)addr;
+        return le32_to_cpu(iSeries_Read_Long(addr));
 }
-EXPORT_SYMBOL(__raw_readl);
 
-extern unsigned long __raw_readq(const volatile void __iomem *addr)
+static u16 iseries_readw_be(const volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        return *(volatile unsigned long __force *)addr;
+        return iSeries_Read_Word(addr);
 }
-EXPORT_SYMBOL(__raw_readq);
 
-extern void __raw_writeb(unsigned char v, volatile void __iomem *addr)
+static u32 iseries_readl_be(const volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        *(volatile unsigned char __force *)addr = v;
+        return iSeries_Read_Long(addr);
 }
-EXPORT_SYMBOL(__raw_writeb);
 
-extern void __raw_writew(unsigned short v, volatile void __iomem *addr)
+static void iseries_writeb(u8 data, volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        *(volatile unsigned short __force *)addr = v;
+        iSeries_Write_Byte(data, addr);
 }
-EXPORT_SYMBOL(__raw_writew);
 
-extern void __raw_writel(unsigned int v, volatile void __iomem *addr)
+static void iseries_writew(u16 data, volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        *(volatile unsigned int __force *)addr = v;
+        iSeries_Write_Word(cpu_to_le16(data), addr);
 }
-EXPORT_SYMBOL(__raw_writel);
 
-extern void __raw_writeq(unsigned long v, volatile void __iomem *addr)
+static void iseries_writel(u32 data, volatile void __iomem *addr)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        *(volatile unsigned long __force *)addr = v;
+        iSeries_Write_Long(cpu_to_le32(data), addr);
 }
-EXPORT_SYMBOL(__raw_writeq);
 
-int in_8(const volatile unsigned char __iomem *addr)
+static void iseries_writew_be(u16 data, volatile void __iomem *addr)
 {
-        if (firmware_has_feature(FW_FEATURE_ISERIES))
-                return iSeries_Read_Byte(addr);
-        return __in_8(addr);
+        iSeries_Write_Word(data, addr);
 }
-EXPORT_SYMBOL(in_8);
 
-void out_8(volatile unsigned char __iomem *addr, int val)
+static void iseries_writel_be(u32 data, volatile void __iomem *addr)
 {
-        if (firmware_has_feature(FW_FEATURE_ISERIES))
-                iSeries_Write_Byte(val, addr);
-        else
-                __out_8(addr, val);
+        iSeries_Write_Long(data, addr);
 }
-EXPORT_SYMBOL(out_8);
 
-int in_le16(const volatile unsigned short __iomem *addr)
+static void iseries_readsb(const volatile void __iomem *addr, void *buf,
+                           unsigned long count)
 {
-        if (firmware_has_feature(FW_FEATURE_ISERIES))
-                return iSeries_Read_Word(addr);
-        return __in_le16(addr);
+        u8 *dst = buf;
+        while(count-- > 0)
+                *(dst++) = iSeries_Read_Byte(addr);
 }
-EXPORT_SYMBOL(in_le16);
 
-int in_be16(const volatile unsigned short __iomem *addr)
+static void iseries_readsw(const volatile void __iomem *addr, void *buf,
+                           unsigned long count)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        return __in_be16(addr);
+        u16 *dst = buf;
+        while(count-- > 0)
+                *(dst++) = iSeries_Read_Word(addr);
 }
-EXPORT_SYMBOL(in_be16);
 
-void out_le16(volatile unsigned short __iomem *addr, int val)
+static void iseries_readsl(const volatile void __iomem *addr, void *buf,
+                           unsigned long count)
 {
-        if (firmware_has_feature(FW_FEATURE_ISERIES))
-                iSeries_Write_Word(val, addr);
-        else
-                __out_le16(addr, val);
+        u32 *dst = buf;
+        while(count-- > 0)
+                *(dst++) = iSeries_Read_Long(addr);
 }
-EXPORT_SYMBOL(out_le16);
 
-void out_be16(volatile unsigned short __iomem *addr, int val)
+static void iseries_writesb(volatile void __iomem *addr, const void *buf,
+                            unsigned long count)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        __out_be16(addr, val);
+        const u8 *src = buf;
+        while(count-- > 0)
+                iSeries_Write_Byte(*(src++), addr);
 }
-EXPORT_SYMBOL(out_be16);
 
-unsigned in_le32(const volatile unsigned __iomem *addr)
+static void iseries_writesw(volatile void __iomem *addr, const void *buf,
+                            unsigned long count)
 {
-        if (firmware_has_feature(FW_FEATURE_ISERIES))
-                return iSeries_Read_Long(addr);
-        return __in_le32(addr);
+        const u16 *src = buf;
+        while(count-- > 0)
+                iSeries_Write_Word(*(src++), addr);
 }
-EXPORT_SYMBOL(in_le32);
 
-unsigned in_be32(const volatile unsigned __iomem *addr)
+static void iseries_writesl(volatile void __iomem *addr, const void *buf,
+                            unsigned long count)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        return __in_be32(addr);
+        const u32 *src = buf;
+        while(count-- > 0)
+                iSeries_Write_Long(*(src++), addr);
 }
-EXPORT_SYMBOL(in_be32);
 
-void out_le32(volatile unsigned __iomem *addr, int val)
+static void iseries_memset_io(volatile void __iomem *addr, int c,
+                              unsigned long n)
 {
-        if (firmware_has_feature(FW_FEATURE_ISERIES))
-                iSeries_Write_Long(val, addr);
-        else
-                __out_le32(addr, val);
-}
-EXPORT_SYMBOL(out_le32);
+        volatile char __iomem *d = addr;
 
-void out_be32(volatile unsigned __iomem *addr, int val)
-{
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
-
-        __out_be32(addr, val);
+        while (n-- > 0)
+                iSeries_Write_Byte(c, d++);
 }
-EXPORT_SYMBOL(out_be32);
 
-unsigned long in_le64(const volatile unsigned long __iomem *addr)
+static void iseries_memcpy_fromio(void *dest, const volatile void __iomem *src,
+                                  unsigned long n)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
+        char *d = dest;
+        const volatile char __iomem *s = src;
 
-        return __in_le64(addr);
+        while (n-- > 0)
+                *d++ = iSeries_Read_Byte(s++);
 }
-EXPORT_SYMBOL(in_le64);
 
-unsigned long in_be64(const volatile unsigned long __iomem *addr)
+static void iseries_memcpy_toio(volatile void __iomem *dest, const void *src,
+                                unsigned long n)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
+        const char *s = src;
+        volatile char __iomem *d = dest;
 
-        return __in_be64(addr);
+        while (n-- > 0)
+                iSeries_Write_Byte(*s++, d++);
 }
-EXPORT_SYMBOL(in_be64);
-
-void out_le64(volatile unsigned long __iomem *addr, unsigned long val)
-{
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
 
-        __out_le64(addr, val);
-}
-EXPORT_SYMBOL(out_le64);
+/* We only set MMIO ops. The default PIO ops will be default
+ * to the MMIO ops + pci_io_base which is 0 on iSeries as
+ * expected so both should work.
+ *
+ * Note that we don't implement the readq/writeq versions as
+ * I don't know of an HV call for doing so. Thus, the default
+ * operation will be used instead, which will fault a the value
+ * return by iSeries for MMIO addresses always hits a non mapped
+ * area. This is as good as the BUG() we used to have there.
+ */
+static struct ppc_pci_io __initdata iseries_pci_io = {
+        .readb = iseries_readb,
+        .readw = iseries_readw,
+        .readl = iseries_readl,
+        .readw_be = iseries_readw_be,
+        .readl_be = iseries_readl_be,
+        .writeb = iseries_writeb,
+        .writew = iseries_writew,
+        .writel = iseries_writel,
+        .writew_be = iseries_writew_be,
+        .writel_be = iseries_writel_be,
+        .readsb = iseries_readsb,
+        .readsw = iseries_readsw,
+        .readsl = iseries_readsl,
+        .writesb = iseries_writesb,
+        .writesw = iseries_writesw,
+        .writesl = iseries_writesl,
+        .memset_io = iseries_memset_io,
+        .memcpy_fromio = iseries_memcpy_fromio,
+        .memcpy_toio = iseries_memcpy_toio,
+};
 
-void out_be64(volatile unsigned long __iomem *addr, unsigned long val)
+/*
+ * iSeries_pcibios_init
+ *
+ * Description:
+ *   This function checks for all possible system PCI host bridges that connect
+ *   PCI buses.  The system hypervisor is queried as to the guest partition
+ *   ownership status.  A pci_controller is built for any bus which is partially
+ *   owned or fully owned by this guest partition.
+ */
+void __init iSeries_pcibios_init(void)
 {
-        BUG_ON(firmware_has_feature(FW_FEATURE_ISERIES));
+        struct pci_controller *phb;
+        struct device_node *root = of_find_node_by_path("/");
+        struct device_node *node = NULL;
 
-        __out_be64(addr, val);
-}
-EXPORT_SYMBOL(out_be64);
+        /* Install IO hooks */
+        ppc_pci_io = iseries_pci_io;
 
-void memset_io(volatile void __iomem *addr, int c, unsigned long n)
-{
-        if (firmware_has_feature(FW_FEATURE_ISERIES)) {
-                volatile char __iomem *d = addr;
+        if (root == NULL) {
+                printk(KERN_CRIT "iSeries_pcibios_init: can't find root "
+                                "of device tree\n");
+                return;
+        }
+        while ((node = of_get_next_child(root, node)) != NULL) {
+                HvBusNumber bus;
+                const u32 *busp;
 
-                while (n-- > 0) {
-                        iSeries_Write_Byte(c, d++);
-                }
-        } else
-                eeh_memset_io(addr, c, n);
-}
-EXPORT_SYMBOL(memset_io);
+                if ((node->type == NULL) || (strcmp(node->type, "pci") != 0))
+                        continue;
 
-void memcpy_fromio(void *dest, const volatile void __iomem *src,
-                                 unsigned long n)
-{
-        if (firmware_has_feature(FW_FEATURE_ISERIES)) {
-                char *d = dest;
-                const volatile char __iomem *s = src;
+                busp = get_property(node, "bus-range", NULL);
+                if (busp == NULL)
+                        continue;
+                bus = *busp;
+                printk("bus %d appears to exist\n", bus);
+                phb = pcibios_alloc_controller(node);
+                if (phb == NULL)
+                        continue;
 
-                while (n-- > 0) {
-                        *d++ = iSeries_Read_Byte(s++);
-                }
-        } else
-                eeh_memcpy_fromio(dest, src, n);
-}
-EXPORT_SYMBOL(memcpy_fromio);
+                phb->pci_mem_offset = phb->local_number = bus;
+                phb->first_busno = bus;
+                phb->last_busno = bus;
+                phb->ops = &iSeries_pci_ops;
+        }
 
-void memcpy_toio(volatile void __iomem *dest, const void *src, unsigned long n)
-{
-        if (firmware_has_feature(FW_FEATURE_ISERIES)) {
-                const char *s = src;
-                volatile char __iomem *d = dest;
+        of_node_put(root);
 
-                while (n-- > 0) {
-                        iSeries_Write_Byte(*s++, d++);
-                }
-        } else
-                eeh_memcpy_toio(dest, src, n);
+        pci_devs_phb_init();
 }
-EXPORT_SYMBOL(memcpy_toio);
+