1 files changed, 30 insertions, 3 deletions

diff --git a/arch/x86/lib/iomem.c b/arch/x86/lib/iomem.c
index 66894675f3c8..df50451d94ef 100644
--- a/arch/x86/lib/iomem.c
+++ b/arch/x86/lib/iomem.c
@@ -2,8 +2,11 @@
 #include <linux/module.h>
 #include <linux/io.h>
 
+#define movs(type,to,from) \
+        asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory")
+
 /* Originally from i386/string.h */
-static __always_inline void __iomem_memcpy(void *to, const void *from, size_t n)
+static __always_inline void rep_movs(void *to, const void *from, size_t n)
 {
         unsigned long d0, d1, d2;
         asm volatile("rep ; movsl\n\t"
@@ -21,13 +24,37 @@ static __always_inline void __iomem_memcpy(void *to, const void *from, size_t n)
 
 void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
 {
-        __iomem_memcpy(to, (const void *)from, n);
+        if (unlikely(!n))
+                return;
+
+        /* Align any unaligned source IO */
+        if (unlikely(1 & (unsigned long)from)) {
+                movs("b", to, from);
+                n--;
+        }
+        if (n > 1 && unlikely(2 & (unsigned long)from)) {
+                movs("w", to, from);
+                n-=2;
+        }
+        rep_movs(to, (const void *)from, n);
 }
 EXPORT_SYMBOL(memcpy_fromio);
 
 void memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
 {
-        __iomem_memcpy((void *)to, (const void *) from, n);
+        if (unlikely(!n))
+                return;
+
+        /* Align any unaligned destination IO */
+        if (unlikely(1 & (unsigned long)to)) {
+                movs("b", to, from);
+                n--;
+        }
+        if (n > 1 && unlikely(2 & (unsigned long)to)) {
+                movs("w", to, from);
+                n-=2;
+        }
+        rep_movs((void *)to, (const void *) from, n);
 }
 EXPORT_SYMBOL(memcpy_toio);