8 files changed, 313 insertions, 144 deletions

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 28e9d6c98941..e458782f3c52 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -972,7 +972,7 @@ config RCU_CPU_STALL_TIMEOUT
         int "RCU CPU stall timeout in seconds"
         depends on TREE_RCU || TREE_PREEMPT_RCU
         range 3 300
-        default 60
+        default 21
         help
           If a given RCU grace period extends more than the specified
           number of seconds, a CPU stall warning is printed.  If the
@@ -1115,7 +1115,7 @@ config NOTIFIER_ERROR_INJECTION
         depends on DEBUG_KERNEL
         select DEBUG_FS
         help
-          This option provides the ability to inject artifical errors to
+          This option provides the ability to inject artificial errors to
           specified notifier chain callbacks. It is useful to test the error
           handling of notifier call chain failures.
 
@@ -1126,7 +1126,7 @@ config CPU_NOTIFIER_ERROR_INJECT
         depends on HOTPLUG_CPU && NOTIFIER_ERROR_INJECTION
         help
           This option provides a kernel module that can be used to test
-          the error handling of the cpu notifiers by injecting artifical
+          the error handling of the cpu notifiers by injecting artificial
           errors to CPU notifier chain callbacks.  It is controlled through
           debugfs interface under /sys/kernel/debug/notifier-error-inject/cpu
 
@@ -1150,7 +1150,7 @@ config PM_NOTIFIER_ERROR_INJECT
         depends on PM && NOTIFIER_ERROR_INJECTION
         default m if PM_DEBUG
         help
-          This option provides the ability to inject artifical errors to
+          This option provides the ability to inject artificial errors to
           PM notifier chain callbacks.  It is controlled through debugfs
           interface /sys/kernel/debug/notifier-error-inject/pm
 
@@ -1173,7 +1173,7 @@ config MEMORY_NOTIFIER_ERROR_INJECT
         tristate "Memory hotplug notifier error injection module"
         depends on MEMORY_HOTPLUG_SPARSE && NOTIFIER_ERROR_INJECTION
         help
-          This option provides the ability to inject artifical errors to
+          This option provides the ability to inject artificial errors to
           memory hotplug notifier chain callbacks.  It is controlled through
           debugfs interface under /sys/kernel/debug/notifier-error-inject/memory
 
@@ -1196,7 +1196,7 @@ config PSERIES_RECONFIG_NOTIFIER_ERROR_INJECT
         tristate "pSeries reconfig notifier error injection module"
         depends on PPC_PSERIES && NOTIFIER_ERROR_INJECTION
         help
-          This option provides the ability to inject artifical errors to
+          This option provides the ability to inject artificial errors to
           pSeries reconfig notifier chain callbacks.  It is controlled
           through debugfs interface under
           /sys/kernel/debug/notifier-error-inject/pSeries-reconfig/
diff --git a/lib/Makefile b/lib/Makefile
index 821a16229111..e2152fa7ff4d 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -12,7 +12,8 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \
          idr.o int_sqrt.o extable.o \
          sha1.o md5.o irq_regs.o reciprocal_div.o argv_split.o \
          proportions.o flex_proportions.o prio_heap.o ratelimit.o show_mem.o \
-         is_single_threaded.o plist.o decompress.o
+         is_single_threaded.o plist.o decompress.o kobject_uevent.o \
+         earlycpio.o
 
 lib-$(CONFIG_MMU) += ioremap.o
 lib-$(CONFIG_SMP) += cpumask.o
@@ -31,7 +32,6 @@ CFLAGS_kobject.o += -DDEBUG
 CFLAGS_kobject_uevent.o += -DDEBUG
 endif
 
-lib-$(CONFIG_HOTPLUG) += kobject_uevent.o
 obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
 obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o
 obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
@@ -163,7 +163,7 @@ $(obj)/crc32table.h: $(obj)/gen_crc32table
 #
 obj-$(CONFIG_OID_REGISTRY) += oid_registry.o
 
-$(obj)/oid_registry.c: $(obj)/oid_registry_data.c
+$(obj)/oid_registry.o: $(obj)/oid_registry_data.c
 
 $(obj)/oid_registry_data.c: $(srctree)/include/linux/oid_registry.h \
                             $(src)/build_OID_registry
diff --git a/lib/asn1_decoder.c b/lib/asn1_decoder.c
index de2c8b5a715b..5293d2433029 100644
--- a/lib/asn1_decoder.c
+++ b/lib/asn1_decoder.c
@@ -91,7 +91,7 @@ next_tag:
 
         /* Extract the length */
         len = data[dp++];
-        if (len < 0x7f) {
+        if (len <= 0x7f) {
                 dp += len;
                 goto next_tag;
         }
diff --git a/lib/bitmap.c b/lib/bitmap.c
index 06fdfa1aeba7..06f7e4fe8d2d 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -353,7 +353,7 @@ again:
 EXPORT_SYMBOL(bitmap_find_next_zero_area);
 
 /*
- * Bitmap printing & parsing functions: first version by Bill Irwin,
+ * Bitmap printing & parsing functions: first version by Nadia Yvette Chambers,
  * second version by Paul Jackson, third by Joe Korty.
  */
 
diff --git a/lib/cpumask.c b/lib/cpumask.c
index 402a54ac35cb..d327b87c99b7 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -161,6 +161,6 @@ EXPORT_SYMBOL(free_cpumask_var);
  */
 void __init free_bootmem_cpumask_var(cpumask_var_t mask)
 {
-        free_bootmem((unsigned long)mask, cpumask_size());
+        free_bootmem(__pa(mask), cpumask_size());
 }
 #endif
diff --git a/lib/earlycpio.c b/lib/earlycpio.c
new file mode 100644
index 000000000000..8078ef49cb79
--- /dev/null
+++ b/lib/earlycpio.c
@@ -0,0 +1,145 @@
+/* ----------------------------------------------------------------------- *
+ *
+ *   Copyright 2012 Intel Corporation; author H. Peter Anvin
+ *
+ *   This file is part of the Linux kernel, and is made available
+ *   under the terms of the GNU General Public License version 2, as
+ *   published by the Free Software Foundation.
+ *
+ *   This program is distributed in the hope it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *   General Public License for more details.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * earlycpio.c
+ *
+ * Find a specific cpio member; must precede any compressed content.
+ * This is used to locate data items in the initramfs used by the
+ * kernel itself during early boot (before the main initramfs is
+ * decompressed.)  It is the responsibility of the initramfs creator
+ * to ensure that these items are uncompressed at the head of the
+ * blob.  Depending on the boot loader or package tool that may be a
+ * separate file or part of the same file.
+ */
+
+#include <linux/earlycpio.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+
+enum cpio_fields {
+        C_MAGIC,
+        C_INO,
+        C_MODE,
+        C_UID,
+        C_GID,
+        C_NLINK,
+        C_MTIME,
+        C_FILESIZE,
+        C_MAJ,
+        C_MIN,
+        C_RMAJ,
+        C_RMIN,
+        C_NAMESIZE,
+        C_CHKSUM,
+        C_NFIELDS
+};
+
+/**
+ * cpio_data find_cpio_data - Search for files in an uncompressed cpio
+ * @path:   The directory to search for, including a slash at the end
+ * @data:   Pointer to the the cpio archive or a header inside
+ * @len:    Remaining length of the cpio based on data pointer
+ * @offset: When a matching file is found, this is the offset to the
+ *          beginning of the cpio. It can be used to iterate through
+ *          the cpio to find all files inside of a directory path
+ *
+ * @return: struct cpio_data containing the address, length and
+ *          filename (with the directory path cut off) of the found file.
+ *          If you search for a filename and not for files in a directory,
+ *          pass the absolute path of the filename in the cpio and make sure
+ *          the match returned an empty filename string.
+ */
+
+struct cpio_data __cpuinit find_cpio_data(const char *path, void *data,
+                                          size_t len,  long *offset)
+{
+        const size_t cpio_header_len = 8*C_NFIELDS - 2;
+        struct cpio_data cd = { NULL, 0, "" };
+        const char *p, *dptr, *nptr;
+        unsigned int ch[C_NFIELDS], *chp, v;
+        unsigned char c, x;
+        size_t mypathsize = strlen(path);
+        int i, j;
+
+        p = data;
+
+        while (len > cpio_header_len) {
+                if (!*p) {
+                        /* All cpio headers need to be 4-byte aligned */
+                        p += 4;
+                        len -= 4;
+                        continue;
+                }
+
+                j = 6;          /* The magic field is only 6 characters */
+                chp = ch;
+                for (i = C_NFIELDS; i; i--) {
+                        v = 0;
+                        while (j--) {
+                                v <<= 4;
+                                c = *p++;
+
+                                x = c - '0';
+                                if (x < 10) {
+                                        v += x;
+                                        continue;
+                                }
+
+                                x = (c | 0x20) - 'a';
+                                if (x < 6) {
+                                        v += x + 10;
+                                        continue;
+                                }
+
+                                goto quit; /* Invalid hexadecimal */
+                        }
+                        *chp++ = v;
+                        j = 8;  /* All other fields are 8 characters */
+                }
+
+                if ((ch[C_MAGIC] - 0x070701) > 1)
+                        goto quit; /* Invalid magic */
+
+                len -= cpio_header_len;
+
+                dptr = PTR_ALIGN(p + ch[C_NAMESIZE], 4);
+                nptr = PTR_ALIGN(dptr + ch[C_FILESIZE], 4);
+
+                if (nptr > p + len || dptr < p || nptr < dptr)
+                        goto quit; /* Buffer overrun */
+
+                if ((ch[C_MODE] & 0170000) == 0100000 &&
+                    ch[C_NAMESIZE] >= mypathsize &&
+                    !memcmp(p, path, mypathsize)) {
+                        *offset = (long)nptr - (long)data;
+                        if (ch[C_NAMESIZE] - mypathsize >= MAX_CPIO_FILE_NAME) {
+                                pr_warn(
+                                "File %s exceeding MAX_CPIO_FILE_NAME [%d]\n",
+                                p, MAX_CPIO_FILE_NAME);
+                        }
+                        strlcpy(cd.name, p + mypathsize, MAX_CPIO_FILE_NAME);
+
+                        cd.data = (void *)dptr;
+                        cd.size = ch[C_FILESIZE];
+                        return cd; /* Found it! */
+                }
+                len -= (nptr - p);
+                p = nptr;
+        }
+
+quit:
+        return cd;
+}
diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h
index 678ce4f1e124..095ab157a521 100644
--- a/lib/mpi/longlong.h
+++ b/lib/mpi/longlong.h
@@ -641,7 +641,14 @@ do { \
         **************  MIPS  *****************
         ***************************************/
 #if defined(__mips__) && W_TYPE_SIZE == 32
-#if __GNUC__ > 2 || __GNUC_MINOR__ >= 7
+#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 4
+#define umul_ppmm(w1, w0, u, v)                 \
+do {                                            \
+        UDItype __ll = (UDItype)(u) * (v);      \
+        w1 = __ll >> 32;                        \
+        w0 = __ll;                              \
+} while (0)
+#elif __GNUC__ > 2 || __GNUC_MINOR__ >= 7
 #define umul_ppmm(w1, w0, u, v) \
         __asm__ ("multu %2,%3" \
         : "=l" ((USItype)(w0)), \
@@ -666,7 +673,15 @@ do { \
         **************  MIPS/64  **************
         ***************************************/
 #if (defined(__mips) && __mips >= 3) && W_TYPE_SIZE == 64
-#if __GNUC__ > 2 || __GNUC_MINOR__ >= 7
+#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 4
+#define umul_ppmm(w1, w0, u, v) \
+do {                                                                    \
+        typedef unsigned int __ll_UTItype __attribute__((mode(TI)));    \
+        __ll_UTItype __ll = (__ll_UTItype)(u) * (v);                    \
+        w1 = __ll >> 64;                                                \
+        w0 = __ll;                                                      \
+} while (0)
+#elif __GNUC__ > 2 || __GNUC_MINOR__ >= 7
 #define umul_ppmm(w1, w0, u, v) \
         __asm__ ("dmultu %2,%3" \
         : "=l" ((UDItype)(w0)), \
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index f114bf6a8e13..196b06984dec 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -57,7 +57,7 @@ int swiotlb_force;
  * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
  * API.
  */
-static char *io_tlb_start, *io_tlb_end;
+static phys_addr_t io_tlb_start, io_tlb_end;
 
 /*
  * The number of IO TLB blocks (in groups of 64) between io_tlb_start and
@@ -70,7 +70,7 @@ static unsigned long io_tlb_nslabs;
  */
 static unsigned long io_tlb_overflow = 32*1024;
 
-static void *io_tlb_overflow_buffer;
+static phys_addr_t io_tlb_overflow_buffer;
 
 /*
  * This is a free list describing the number of free entries available from
@@ -125,27 +125,38 @@ static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
 void swiotlb_print_info(void)
 {
         unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT;
-        phys_addr_t pstart, pend;
+        unsigned char *vstart, *vend;
 
-        pstart = virt_to_phys(io_tlb_start);
-        pend = virt_to_phys(io_tlb_end);
+        vstart = phys_to_virt(io_tlb_start);
+        vend = phys_to_virt(io_tlb_end);
 
         printk(KERN_INFO "software IO TLB [mem %#010llx-%#010llx] (%luMB) mapped at [%p-%p]\n",
-               (unsigned long long)pstart, (unsigned long long)pend - 1,
-               bytes >> 20, io_tlb_start, io_tlb_end - 1);
+               (unsigned long long)io_tlb_start,
+               (unsigned long long)io_tlb_end,
+               bytes >> 20, vstart, vend - 1);
 }
 
 void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
 {
+        void *v_overflow_buffer;
         unsigned long i, bytes;
 
         bytes = nslabs << IO_TLB_SHIFT;
 
         io_tlb_nslabs = nslabs;
-        io_tlb_start = tlb;
+        io_tlb_start = __pa(tlb);
         io_tlb_end = io_tlb_start + bytes;
 
         /*
+         * Get the overflow emergency buffer
+         */
+        v_overflow_buffer = alloc_bootmem_low_pages(PAGE_ALIGN(io_tlb_overflow));
+        if (!v_overflow_buffer)
+                panic("Cannot allocate SWIOTLB overflow buffer!\n");
+
+        io_tlb_overflow_buffer = __pa(v_overflow_buffer);
+
+        /*
          * Allocate and initialize the free list array.  This array is used
          * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
          * between io_tlb_start and io_tlb_end.
@@ -156,12 +167,6 @@ void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
         io_tlb_index = 0;
         io_tlb_orig_addr = alloc_bootmem_pages(PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
 
-        /*
-         * Get the overflow emergency buffer
-         */
-        io_tlb_overflow_buffer = alloc_bootmem_low_pages(PAGE_ALIGN(io_tlb_overflow));
-        if (!io_tlb_overflow_buffer)
-                panic("Cannot allocate SWIOTLB overflow buffer!\n");
         if (verbose)
                 swiotlb_print_info();
 }
@@ -173,6 +178,7 @@ void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
 static void __init
 swiotlb_init_with_default_size(size_t default_size, int verbose)
 {
+        unsigned char *vstart;
         unsigned long bytes;
 
         if (!io_tlb_nslabs) {
@@ -185,11 +191,11 @@ swiotlb_init_with_default_size(size_t default_size, int verbose)
         /*
          * Get IO TLB memory from the low pages
          */
-        io_tlb_start = alloc_bootmem_low_pages(PAGE_ALIGN(bytes));
-        if (!io_tlb_start)
+        vstart = alloc_bootmem_low_pages(PAGE_ALIGN(bytes));
+        if (!vstart)
                 panic("Cannot allocate SWIOTLB buffer");
 
-        swiotlb_init_with_tbl(io_tlb_start, io_tlb_nslabs, verbose);
+        swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose);
 }
 
 void __init
@@ -207,6 +213,7 @@ int
 swiotlb_late_init_with_default_size(size_t default_size)
 {
         unsigned long bytes, req_nslabs = io_tlb_nslabs;
+        unsigned char *vstart = NULL;
         unsigned int order;
         int rc = 0;
 
@@ -223,14 +230,14 @@ swiotlb_late_init_with_default_size(size_t default_size)
         bytes = io_tlb_nslabs << IO_TLB_SHIFT;
 
         while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
-                io_tlb_start = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
-                                                        order);
-                if (io_tlb_start)
+                vstart = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
+                                                  order);
+                if (vstart)
                         break;
                 order--;
         }
 
-        if (!io_tlb_start) {
+        if (!vstart) {
                 io_tlb_nslabs = req_nslabs;
                 return -ENOMEM;
         }
@@ -239,9 +246,9 @@ swiotlb_late_init_with_default_size(size_t default_size)
                        "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
                 io_tlb_nslabs = SLABS_PER_PAGE << order;
         }
-        rc = swiotlb_late_init_with_tbl(io_tlb_start, io_tlb_nslabs);
+        rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs);
         if (rc)
-                free_pages((unsigned long)io_tlb_start, order);
+                free_pages((unsigned long)vstart, order);
         return rc;
 }
 
@@ -249,14 +256,25 @@ int
 swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
 {
         unsigned long i, bytes;
+        unsigned char *v_overflow_buffer;
 
         bytes = nslabs << IO_TLB_SHIFT;
 
         io_tlb_nslabs = nslabs;
-        io_tlb_start = tlb;
+        io_tlb_start = virt_to_phys(tlb);
         io_tlb_end = io_tlb_start + bytes;
 
-        memset(io_tlb_start, 0, bytes);
+        memset(tlb, 0, bytes);
+
+        /*
+         * Get the overflow emergency buffer
+         */
+        v_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
+                                                     get_order(io_tlb_overflow));
+        if (!v_overflow_buffer)
+                goto cleanup2;
+
+        io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer);
 
         /*
          * Allocate and initialize the free list array.  This array is used
@@ -266,7 +284,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
         io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
                                       get_order(io_tlb_nslabs * sizeof(int)));
         if (!io_tlb_list)
-                goto cleanup2;
+                goto cleanup3;
 
         for (i = 0; i < io_tlb_nslabs; i++)
                 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
@@ -277,18 +295,10 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
                                  get_order(io_tlb_nslabs *
                                            sizeof(phys_addr_t)));
         if (!io_tlb_orig_addr)
-                goto cleanup3;
+                goto cleanup4;
 
         memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t));
 
-        /*
-         * Get the overflow emergency buffer
-         */
-        io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
-                                                  get_order(io_tlb_overflow));
-        if (!io_tlb_overflow_buffer)
-                goto cleanup4;
-
         swiotlb_print_info();
 
         late_alloc = 1;
@@ -296,42 +306,42 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
         return 0;
 
 cleanup4:
-        free_pages((unsigned long)io_tlb_orig_addr,
-                   get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
-        io_tlb_orig_addr = NULL;
-cleanup3:
         free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
                                                          sizeof(int)));
         io_tlb_list = NULL;
+cleanup3:
+        free_pages((unsigned long)v_overflow_buffer,
+                   get_order(io_tlb_overflow));
+        io_tlb_overflow_buffer = 0;
 cleanup2:
-        io_tlb_end = NULL;
-        io_tlb_start = NULL;
+        io_tlb_end = 0;
+        io_tlb_start = 0;
         io_tlb_nslabs = 0;
         return -ENOMEM;
 }
 
 void __init swiotlb_free(void)
 {
-        if (!io_tlb_overflow_buffer)
+        if (!io_tlb_orig_addr)
                 return;
 
         if (late_alloc) {
-                free_pages((unsigned long)io_tlb_overflow_buffer,
+                free_pages((unsigned long)phys_to_virt(io_tlb_overflow_buffer),
                            get_order(io_tlb_overflow));
                 free_pages((unsigned long)io_tlb_orig_addr,
                            get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
                 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
                                                                  sizeof(int)));
-                free_pages((unsigned long)io_tlb_start,
+                free_pages((unsigned long)phys_to_virt(io_tlb_start),
                            get_order(io_tlb_nslabs << IO_TLB_SHIFT));
         } else {
-                free_bootmem_late(__pa(io_tlb_overflow_buffer),
+                free_bootmem_late(io_tlb_overflow_buffer,
                                   PAGE_ALIGN(io_tlb_overflow));
                 free_bootmem_late(__pa(io_tlb_orig_addr),
                                   PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
                 free_bootmem_late(__pa(io_tlb_list),
                                   PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
-                free_bootmem_late(__pa(io_tlb_start),
+                free_bootmem_late(io_tlb_start,
                                   PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
         }
         io_tlb_nslabs = 0;
@@ -339,21 +349,21 @@ void __init swiotlb_free(void)
 
 static int is_swiotlb_buffer(phys_addr_t paddr)
 {
-        return paddr >= virt_to_phys(io_tlb_start) &&
-                paddr < virt_to_phys(io_tlb_end);
+        return paddr >= io_tlb_start && paddr < io_tlb_end;
 }
 
 /*
  * Bounce: copy the swiotlb buffer back to the original dma location
  */
-void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
-                    enum dma_data_direction dir)
+static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
+                           size_t size, enum dma_data_direction dir)
 {
-        unsigned long pfn = PFN_DOWN(phys);
+        unsigned long pfn = PFN_DOWN(orig_addr);
+        unsigned char *vaddr = phys_to_virt(tlb_addr);
 
         if (PageHighMem(pfn_to_page(pfn))) {
                 /* The buffer does not have a mapping.  Map it in and copy */
-                unsigned int offset = phys & ~PAGE_MASK;
+                unsigned int offset = orig_addr & ~PAGE_MASK;
                 char *buffer;
                 unsigned int sz = 0;
                 unsigned long flags;
@@ -364,32 +374,31 @@ void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
                         local_irq_save(flags);
                         buffer = kmap_atomic(pfn_to_page(pfn));
                         if (dir == DMA_TO_DEVICE)
-                                memcpy(dma_addr, buffer + offset, sz);
+                                memcpy(vaddr, buffer + offset, sz);
                         else
-                                memcpy(buffer + offset, dma_addr, sz);
+                                memcpy(buffer + offset, vaddr, sz);
                         kunmap_atomic(buffer);
                         local_irq_restore(flags);
 
                         size -= sz;
                         pfn++;
-                        dma_addr += sz;
+                        vaddr += sz;
                         offset = 0;
                 }
+        } else if (dir == DMA_TO_DEVICE) {
+                memcpy(vaddr, phys_to_virt(orig_addr), size);
         } else {
-                if (dir == DMA_TO_DEVICE)
-                        memcpy(dma_addr, phys_to_virt(phys), size);
-                else
-                        memcpy(phys_to_virt(phys), dma_addr, size);
+                memcpy(phys_to_virt(orig_addr), vaddr, size);
         }
 }
-EXPORT_SYMBOL_GPL(swiotlb_bounce);
 
-void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,
-                             phys_addr_t phys, size_t size,
-                             enum dma_data_direction dir)
+phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
+                                   dma_addr_t tbl_dma_addr,
+                                   phys_addr_t orig_addr, size_t size,
+                                   enum dma_data_direction dir)
 {
         unsigned long flags;
-        char *dma_addr;
+        phys_addr_t tlb_addr;
         unsigned int nslots, stride, index, wrap;
         int i;
         unsigned long mask;
@@ -453,7 +462,7 @@ void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,
                                 io_tlb_list[i] = 0;
                         for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
                                 io_tlb_list[i] = ++count;
-                        dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
+                        tlb_addr = io_tlb_start + (index << IO_TLB_SHIFT);
 
                         /*
                          * Update the indices to avoid searching in the next
@@ -471,7 +480,7 @@ void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,
 
 not_found:
         spin_unlock_irqrestore(&io_tlb_lock, flags);
-        return NULL;
+        return SWIOTLB_MAP_ERROR;
 found:
         spin_unlock_irqrestore(&io_tlb_lock, flags);
 
@@ -481,11 +490,11 @@ found:
          * needed.
          */
         for (i = 0; i < nslots; i++)
-                io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);
+                io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT);
         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
-                swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
+                swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE);
 
-        return dma_addr;
+        return tlb_addr;
 }
 EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
 
@@ -493,11 +502,10 @@ EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
  * Allocates bounce buffer and returns its kernel virtual address.
  */
 
-static void *
-map_single(struct device *hwdev, phys_addr_t phys, size_t size,
-           enum dma_data_direction dir)
+phys_addr_t map_single(struct device *hwdev, phys_addr_t phys, size_t size,
+                       enum dma_data_direction dir)
 {
-        dma_addr_t start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start);
+        dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start);
 
         return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir);
 }
@@ -505,20 +513,19 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,
 /*
  * dma_addr is the kernel virtual address of the bounce buffer to unmap.
  */
-void
-swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size,
-                        enum dma_data_direction dir)
+void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
+                              size_t size, enum dma_data_direction dir)
 {
         unsigned long flags;
         int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
-        int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
-        phys_addr_t phys = io_tlb_orig_addr[index];
+        int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;
+        phys_addr_t orig_addr = io_tlb_orig_addr[index];
 
         /*
          * First, sync the memory before unmapping the entry
          */
-        if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
-                swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
+        if (orig_addr && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
+                swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE);
 
         /*
          * Return the buffer to the free list by setting the corresponding
@@ -547,26 +554,27 @@ swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size,
 }
 EXPORT_SYMBOL_GPL(swiotlb_tbl_unmap_single);
 
-void
-swiotlb_tbl_sync_single(struct device *hwdev, char *dma_addr, size_t size,
-                        enum dma_data_direction dir,
-                        enum dma_sync_target target)
+void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
+                             size_t size, enum dma_data_direction dir,
+                             enum dma_sync_target target)
 {
-        int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
-        phys_addr_t phys = io_tlb_orig_addr[index];
+        int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;
+        phys_addr_t orig_addr = io_tlb_orig_addr[index];
 
-        phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1));
+        orig_addr += (unsigned long)tlb_addr & ((1 << IO_TLB_SHIFT) - 1);
 
         switch (target) {
         case SYNC_FOR_CPU:
                 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
-                        swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
+                        swiotlb_bounce(orig_addr, tlb_addr,
+                                       size, DMA_FROM_DEVICE);
                 else
                         BUG_ON(dir != DMA_TO_DEVICE);
                 break;
         case SYNC_FOR_DEVICE:
                 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
-                        swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
+                        swiotlb_bounce(orig_addr, tlb_addr,
+                                       size, DMA_TO_DEVICE);
                 else
                         BUG_ON(dir != DMA_FROM_DEVICE);
                 break;
@@ -589,12 +597,15 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
                 dma_mask = hwdev->coherent_dma_mask;
 
         ret = (void *)__get_free_pages(flags, order);
-        if (ret && swiotlb_virt_to_bus(hwdev, ret) + size - 1 > dma_mask) {
-                /*
-                 * The allocated memory isn't reachable by the device.
-                 */
-                free_pages((unsigned long) ret, order);
-                ret = NULL;
+        if (ret) {
+                dev_addr = swiotlb_virt_to_bus(hwdev, ret);
+                if (dev_addr + size - 1 > dma_mask) {
+                        /*
+                         * The allocated memory isn't reachable by the device.
+                         */
+                        free_pages((unsigned long) ret, order);
+                        ret = NULL;
+                }
         }
         if (!ret) {
                 /*
@@ -602,25 +613,29 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
                  * GFP_DMA memory; fall back on map_single(), which
                  * will grab memory from the lowest available address range.
                  */
-                ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
-                if (!ret)
+                phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
+                if (paddr == SWIOTLB_MAP_ERROR)
                         return NULL;
-        }
 
-        memset(ret, 0, size);
-        dev_addr = swiotlb_virt_to_bus(hwdev, ret);
+                ret = phys_to_virt(paddr);
+                dev_addr = phys_to_dma(hwdev, paddr);
 
-        /* Confirm address can be DMA'd by device */
-        if (dev_addr + size - 1 > dma_mask) {
-                printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
-                       (unsigned long long)dma_mask,
-                       (unsigned long long)dev_addr);
+                /* Confirm address can be DMA'd by device */
+                if (dev_addr + size - 1 > dma_mask) {
+                        printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
+                               (unsigned long long)dma_mask,
+                               (unsigned long long)dev_addr);
 
-                /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
-                swiotlb_tbl_unmap_single(hwdev, ret, size, DMA_TO_DEVICE);
-                return NULL;
+                        /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
+                        swiotlb_tbl_unmap_single(hwdev, paddr,
+                                                 size, DMA_TO_DEVICE);
+                        return NULL;
+                }
         }
+
         *dma_handle = dev_addr;
+        memset(ret, 0, size);
+
         return ret;
 }
 EXPORT_SYMBOL(swiotlb_alloc_coherent);
@@ -636,7 +651,7 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
                 free_pages((unsigned long)vaddr, get_order(size));
         else
                 /* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */
-                swiotlb_tbl_unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
+                swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE);
 }
 EXPORT_SYMBOL(swiotlb_free_coherent);
 
@@ -677,9 +692,8 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
                             enum dma_data_direction dir,
                             struct dma_attrs *attrs)
 {
-        phys_addr_t phys = page_to_phys(page) + offset;
+        phys_addr_t map, phys = page_to_phys(page) + offset;
         dma_addr_t dev_addr = phys_to_dma(dev, phys);
-        void *map;
 
         BUG_ON(dir == DMA_NONE);
         /*
@@ -690,23 +704,19 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
         if (dma_capable(dev, dev_addr, size) && !swiotlb_force)
                 return dev_addr;
 
-        /*
-         * Oh well, have to allocate and map a bounce buffer.
-         */
+        /* Oh well, have to allocate and map a bounce buffer. */
         map = map_single(dev, phys, size, dir);
-        if (!map) {
+        if (map == SWIOTLB_MAP_ERROR) {
                 swiotlb_full(dev, size, dir, 1);
-                map = io_tlb_overflow_buffer;
+                return phys_to_dma(dev, io_tlb_overflow_buffer);
         }
 
-        dev_addr = swiotlb_virt_to_bus(dev, map);
+        dev_addr = phys_to_dma(dev, map);
 
-        /*
-         * Ensure that the address returned is DMA'ble
-         */
+        /* Ensure that the address returned is DMA'ble */
         if (!dma_capable(dev, dev_addr, size)) {
                 swiotlb_tbl_unmap_single(dev, map, size, dir);
-                dev_addr = swiotlb_virt_to_bus(dev, io_tlb_overflow_buffer);
+                return phys_to_dma(dev, io_tlb_overflow_buffer);
         }
 
         return dev_addr;
@@ -729,7 +739,7 @@ static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,
         BUG_ON(dir == DMA_NONE);
 
         if (is_swiotlb_buffer(paddr)) {
-                swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir);
+                swiotlb_tbl_unmap_single(hwdev, paddr, size, dir);
                 return;
         }
 
@@ -773,8 +783,7 @@ swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
         BUG_ON(dir == DMA_NONE);
 
         if (is_swiotlb_buffer(paddr)) {
-                swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir,
-                                       target);
+                swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);
                 return;
         }
 
@@ -831,9 +840,9 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
 
                 if (swiotlb_force ||
                     !dma_capable(hwdev, dev_addr, sg->length)) {
-                        void *map = map_single(hwdev, sg_phys(sg),
-                                               sg->length, dir);
-                        if (!map) {
+                        phys_addr_t map = map_single(hwdev, sg_phys(sg),
+                                                     sg->length, dir);
+                        if (map == SWIOTLB_MAP_ERROR) {
                                 /* Don't panic here, we expect map_sg users
                                    to do proper error handling. */
                                 swiotlb_full(hwdev, sg->length, dir, 0);
@@ -842,7 +851,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
                                 sgl[0].dma_length = 0;
                                 return 0;
                         }
-                        sg->dma_address = swiotlb_virt_to_bus(hwdev, map);
+                        sg->dma_address = phys_to_dma(hwdev, map);
                 } else
                         sg->dma_address = dev_addr;
                 sg->dma_length = sg->length;
@@ -925,7 +934,7 @@ EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
 int
 swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
 {
-        return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer));
+        return (dma_addr == phys_to_dma(hwdev, io_tlb_overflow_buffer));
 }
 EXPORT_SYMBOL(swiotlb_dma_mapping_error);
 
@@ -938,6 +947,6 @@ EXPORT_SYMBOL(swiotlb_dma_mapping_error);
 int
 swiotlb_dma_supported(struct device *hwdev, u64 mask)
 {
-        return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask;
+        return phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
 }
 EXPORT_SYMBOL(swiotlb_dma_supported);