Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6

140 files changed, 11835 insertions, 4099 deletions

diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index d295bdccc09c..9335b87c5174 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -115,6 +115,9 @@ static const struct file_operations acpi_button_state_fops = {
         .release = single_release,
 };
 
+static BLOCKING_NOTIFIER_HEAD(acpi_lid_notifier);
+static struct acpi_device *lid_device;
+
 /* --------------------------------------------------------------------------
                               FS Interface (/proc)
    -------------------------------------------------------------------------- */
@@ -231,11 +234,38 @@ static int acpi_button_remove_fs(struct acpi_device *device)
 /* --------------------------------------------------------------------------
                                 Driver Interface
    -------------------------------------------------------------------------- */
+int acpi_lid_notifier_register(struct notifier_block *nb)
+{
+        return blocking_notifier_chain_register(&acpi_lid_notifier, nb);
+}
+EXPORT_SYMBOL(acpi_lid_notifier_register);
+
+int acpi_lid_notifier_unregister(struct notifier_block *nb)
+{
+        return blocking_notifier_chain_unregister(&acpi_lid_notifier, nb);
+}
+EXPORT_SYMBOL(acpi_lid_notifier_unregister);
+
+int acpi_lid_open(void)
+{
+        acpi_status status;
+        unsigned long long state;
+
+        status = acpi_evaluate_integer(lid_device->handle, "_LID", NULL,
+                                       &state);
+        if (ACPI_FAILURE(status))
+                return -ENODEV;
+
+        return !!state;
+}
+EXPORT_SYMBOL(acpi_lid_open);
+
 static int acpi_lid_send_state(struct acpi_device *device)
 {
         struct acpi_button *button = acpi_driver_data(device);
         unsigned long long state;
         acpi_status status;
+        int ret;
 
         status = acpi_evaluate_integer(device->handle, "_LID", NULL, &state);
         if (ACPI_FAILURE(status))
@@ -244,7 +274,12 @@ static int acpi_lid_send_state(struct acpi_device *device)
         /* input layer checks if event is redundant */
         input_report_switch(button->input, SW_LID, !state);
         input_sync(button->input);
-        return 0;
+
+        ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, device);
+        if (ret == NOTIFY_DONE)
+                ret = blocking_notifier_call_chain(&acpi_lid_notifier, state,
+                                                   device);
+        return ret;
 }
 
 static void acpi_button_notify(struct acpi_device *device, u32 event)
@@ -366,8 +401,14 @@ static int acpi_button_add(struct acpi_device *device)
         error = input_register_device(input);
         if (error)
                 goto err_remove_fs;
-        if (button->type == ACPI_BUTTON_TYPE_LID)
+        if (button->type == ACPI_BUTTON_TYPE_LID) {
                 acpi_lid_send_state(device);
+                /*
+                 * This assumes there's only one lid device, or if there are
+                 * more we only care about the last one...
+                 */
+                lid_device = device;
+        }
 
         if (device->wakeup.flags.valid) {
                 /* Button's GPE is run-wake GPE */
diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
index 71d1b9bab70b..614da5b8613a 100644
--- a/drivers/cdrom/cdrom.c
+++ b/drivers/cdrom/cdrom.c
@@ -3412,7 +3412,7 @@ static int cdrom_print_info(const char *header, int val, char *info,
         return 0;
 }
 
-static int cdrom_sysctl_info(ctl_table *ctl, int write, struct file * filp,
+static int cdrom_sysctl_info(ctl_table *ctl, int write,
                            void __user *buffer, size_t *lenp, loff_t *ppos)
 {
         int pos;
@@ -3489,7 +3489,7 @@ static int cdrom_sysctl_info(ctl_table *ctl, int write, struct file * filp,
                 goto done;
 doit:
         mutex_unlock(&cdrom_mutex);
-        return proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        return proc_dostring(ctl, write, buffer, lenp, ppos);
 done:
         printk(KERN_INFO "cdrom: info buffer too small\n");
         goto doit;
@@ -3525,12 +3525,12 @@ static void cdrom_update_settings(void)
         mutex_unlock(&cdrom_mutex);
 }
 
-static int cdrom_sysctl_handler(ctl_table *ctl, int write, struct file * filp,
+static int cdrom_sysctl_handler(ctl_table *ctl, int write,
                                 void __user *buffer, size_t *lenp, loff_t *ppos)
 {
         int ret;
         
-        ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
+        ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
 
         if (write) {
         
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index 6a06913b01d3..08a6f50ae791 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -1087,6 +1087,14 @@ config MMTIMER
           The mmtimer device allows direct userspace access to the
           Altix system timer.
 
+config UV_MMTIMER
+        tristate "UV_MMTIMER Memory mapped RTC for SGI UV"
+        depends on X86_UV
+        default m
+        help
+          The uv_mmtimer device allows direct userspace access to the
+          UV system timer.
+
 source "drivers/char/tpm/Kconfig"
 
 config TELCLOCK
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index 66f779ad4f4c..19a79dd79eee 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -58,6 +58,7 @@ obj-$(CONFIG_RAW_DRIVER)	+= raw.o
 obj-$(CONFIG_SGI_SNSC)          += snsc.o snsc_event.o
 obj-$(CONFIG_MSPEC)             += mspec.o
 obj-$(CONFIG_MMTIMER)           += mmtimer.o
+obj-$(CONFIG_UV_MMTIMER)        += uv_mmtimer.o
 obj-$(CONFIG_VIOTAPE)           += viotape.o
 obj-$(CONFIG_HVCS)              += hvcs.o
 obj-$(CONFIG_IBM_BSR)           += bsr.o
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index 1540e693d91e..4068467ce7b9 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -46,6 +46,8 @@
 #define PCI_DEVICE_ID_INTEL_Q35_IG          0x29B2
 #define PCI_DEVICE_ID_INTEL_Q33_HB          0x29D0
 #define PCI_DEVICE_ID_INTEL_Q33_IG          0x29D2
+#define PCI_DEVICE_ID_INTEL_B43_HB          0x2E40
+#define PCI_DEVICE_ID_INTEL_B43_IG          0x2E42
 #define PCI_DEVICE_ID_INTEL_GM45_HB         0x2A40
 #define PCI_DEVICE_ID_INTEL_GM45_IG         0x2A42
 #define PCI_DEVICE_ID_INTEL_IGD_E_HB        0x2E00
@@ -91,6 +93,7 @@
                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_G45_HB || \
                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_GM45_HB || \
                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_G41_HB || \
+                agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_B43_HB || \
                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_IGDNG_D_HB || \
                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_IGDNG_M_HB || \
                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_IGDNG_MA_HB)
@@ -804,23 +807,39 @@ static void intel_i830_setup_flush(void)
         if (!intel_private.i8xx_page)
                 return;
 
-        /* make page uncached */
-        map_page_into_agp(intel_private.i8xx_page);
-
         intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
         if (!intel_private.i8xx_flush_page)
                 intel_i830_fini_flush();
 }
 
+static void
+do_wbinvd(void *null)
+{
+        wbinvd();
+}
+
+/* The chipset_flush interface needs to get data that has already been
+ * flushed out of the CPU all the way out to main memory, because the GPU
+ * doesn't snoop those buffers.
+ *
+ * The 8xx series doesn't have the same lovely interface for flushing the
+ * chipset write buffers that the later chips do. According to the 865
+ * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
+ * that buffer out, we just fill 1KB and clflush it out, on the assumption
+ * that it'll push whatever was in there out.  It appears to work.
+ */
 static void intel_i830_chipset_flush(struct agp_bridge_data *bridge)
 {
         unsigned int *pg = intel_private.i8xx_flush_page;
-        int i;
 
-        for (i = 0; i < 256; i += 2)
-                *(pg + i) = i;
+        memset(pg, 0, 1024);
 
-        wmb();
+        if (cpu_has_clflush) {
+                clflush_cache_range(pg, 1024);
+        } else {
+                if (on_each_cpu(do_wbinvd, NULL, 1) != 0)
+                        printk(KERN_ERR "Timed out waiting for cache flush.\n");
+        }
 }
 
 /* The intel i830 automatically initializes the agp aperture during POST.
@@ -1341,6 +1360,7 @@ static void intel_i965_get_gtt_range(int *gtt_offset, int *gtt_size)
         case PCI_DEVICE_ID_INTEL_Q45_HB:
         case PCI_DEVICE_ID_INTEL_G45_HB:
         case PCI_DEVICE_ID_INTEL_G41_HB:
+        case PCI_DEVICE_ID_INTEL_B43_HB:
         case PCI_DEVICE_ID_INTEL_IGDNG_D_HB:
         case PCI_DEVICE_ID_INTEL_IGDNG_M_HB:
         case PCI_DEVICE_ID_INTEL_IGDNG_MA_HB:
@@ -2335,6 +2355,8 @@ static const struct intel_driver_description {
             "Q45/Q43", NULL, &intel_i965_driver },
         { PCI_DEVICE_ID_INTEL_G45_HB, PCI_DEVICE_ID_INTEL_G45_IG, 0,
             "G45/G43", NULL, &intel_i965_driver },
+        { PCI_DEVICE_ID_INTEL_B43_HB, PCI_DEVICE_ID_INTEL_B43_IG, 0,
+            "B43", NULL, &intel_i965_driver },
         { PCI_DEVICE_ID_INTEL_G41_HB, PCI_DEVICE_ID_INTEL_G41_IG, 0,
             "G41", NULL, &intel_i965_driver },
         { PCI_DEVICE_ID_INTEL_IGDNG_D_HB, PCI_DEVICE_ID_INTEL_IGDNG_D_IG, 0,
@@ -2535,6 +2557,7 @@ static struct pci_device_id agp_intel_pci_table[] = {
         ID(PCI_DEVICE_ID_INTEL_Q45_HB),
         ID(PCI_DEVICE_ID_INTEL_G45_HB),
         ID(PCI_DEVICE_ID_INTEL_G41_HB),
+        ID(PCI_DEVICE_ID_INTEL_B43_HB),
         ID(PCI_DEVICE_ID_INTEL_IGDNG_D_HB),
         ID(PCI_DEVICE_ID_INTEL_IGDNG_M_HB),
         ID(PCI_DEVICE_ID_INTEL_IGDNG_MA_HB),
diff --git a/drivers/char/bfin-otp.c b/drivers/char/bfin-otp.c
index 0a01329451e4..e3dd24bff514 100644
--- a/drivers/char/bfin-otp.c
+++ b/drivers/char/bfin-otp.c
@@ -1,8 +1,7 @@
 /*
  * Blackfin On-Chip OTP Memory Interface
- *  Supports BF52x/BF54x
  *
- * Copyright 2007-2008 Analog Devices Inc.
+ * Copyright 2007-2009 Analog Devices Inc.
  *
  * Enter bugs at http://blackfin.uclinux.org/
  *
@@ -17,8 +16,10 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/types.h>
+#include <mtd/mtd-abi.h>
 
 #include <asm/blackfin.h>
+#include <asm/bfrom.h>
 #include <asm/uaccess.h>
 
 #define stamp(fmt, args...) pr_debug("%s:%i: " fmt "\n", __func__, __LINE__, ## args)
@@ -30,39 +31,6 @@
 
 static DEFINE_MUTEX(bfin_otp_lock);
 
-/* OTP Boot ROM functions */
-#define _BOOTROM_OTP_COMMAND           0xEF000018
-#define _BOOTROM_OTP_READ              0xEF00001A
-#define _BOOTROM_OTP_WRITE             0xEF00001C
-
-static u32 (* const otp_command)(u32 command, u32 value) = (void *)_BOOTROM_OTP_COMMAND;
-static u32 (* const otp_read)(u32 page, u32 flags, u64 *page_content) = (void *)_BOOTROM_OTP_READ;
-static u32 (* const otp_write)(u32 page, u32 flags, u64 *page_content) = (void *)_BOOTROM_OTP_WRITE;
-
-/* otp_command(): defines for "command" */
-#define OTP_INIT             0x00000001
-#define OTP_CLOSE            0x00000002
-
-/* otp_{read,write}(): defines for "flags" */
-#define OTP_LOWER_HALF       0x00000000 /* select upper/lower 64-bit half (bit 0) */
-#define OTP_UPPER_HALF       0x00000001
-#define OTP_NO_ECC           0x00000010 /* do not use ECC */
-#define OTP_LOCK             0x00000020 /* sets page protection bit for page */
-#define OTP_ACCESS_READ      0x00001000
-#define OTP_ACCESS_READWRITE 0x00002000
-
-/* Return values for all functions */
-#define OTP_SUCCESS          0x00000000
-#define OTP_MASTER_ERROR     0x001
-#define OTP_WRITE_ERROR      0x003
-#define OTP_READ_ERROR       0x005
-#define OTP_ACC_VIO_ERROR    0x009
-#define OTP_DATA_MULT_ERROR  0x011
-#define OTP_ECC_MULT_ERROR   0x021
-#define OTP_PREV_WR_ERROR    0x041
-#define OTP_DATA_SB_WARN     0x100
-#define OTP_ECC_SB_WARN      0x200
-
 /**
  *      bfin_otp_read - Read OTP pages
  *
@@ -86,9 +54,11 @@ static ssize_t bfin_otp_read(struct file *file, char __user *buff, size_t count,
         page = *pos / (sizeof(u64) * 2);
         while (bytes_done < count) {
                 flags = (*pos % (sizeof(u64) * 2) ? OTP_UPPER_HALF : OTP_LOWER_HALF);
-                stamp("processing page %i (%s)", page, (flags == OTP_UPPER_HALF ? "upper" : "lower"));
-                ret = otp_read(page, flags, &content);
+                stamp("processing page %i (0x%x:%s)", page, flags,
+                        (flags & OTP_UPPER_HALF ? "upper" : "lower"));
+                ret = bfrom_OtpRead(page, flags, &content);
                 if (ret & OTP_MASTER_ERROR) {
+                        stamp("error from otp: 0x%x", ret);
                         bytes_done = -EIO;
                         break;
                 }
@@ -96,7 +66,7 @@ static ssize_t bfin_otp_read(struct file *file, char __user *buff, size_t count,
                         bytes_done = -EFAULT;
                         break;
                 }
-                if (flags == OTP_UPPER_HALF)
+                if (flags & OTP_UPPER_HALF)
                         ++page;
                 bytes_done += sizeof(content);
                 *pos += sizeof(content);
@@ -108,14 +78,53 @@ static ssize_t bfin_otp_read(struct file *file, char __user *buff, size_t count,
 }
 
 #ifdef CONFIG_BFIN_OTP_WRITE_ENABLE
+static bool allow_writes;
+
+/**
+ *      bfin_otp_init_timing - setup OTP timing parameters
+ *
+ *      Required before doing any write operation.  Algorithms from HRM.
+ */
+static u32 bfin_otp_init_timing(void)
+{
+        u32 tp1, tp2, tp3, timing;
+
+        tp1 = get_sclk() / 1000000;
+        tp2 = (2 * get_sclk() / 10000000) << 8;
+        tp3 = (0x1401) << 15;
+        timing = tp1 | tp2 | tp3;
+        if (bfrom_OtpCommand(OTP_INIT, timing))
+                return 0;
+
+        return timing;
+}
+
+/**
+ *      bfin_otp_deinit_timing - set timings to only allow reads
+ *
+ *      Should be called after all writes are done.
+ */
+static void bfin_otp_deinit_timing(u32 timing)
+{
+        /* mask bits [31:15] so that any attempts to write fail */
+        bfrom_OtpCommand(OTP_CLOSE, 0);
+        bfrom_OtpCommand(OTP_INIT, timing & ~(-1 << 15));
+        bfrom_OtpCommand(OTP_CLOSE, 0);
+}
+
 /**
- *      bfin_otp_write - Write OTP pages
+ *      bfin_otp_write - write OTP pages
  *
  *      All writes must be in half page chunks (half page == 64 bits).
  */
 static ssize_t bfin_otp_write(struct file *filp, const char __user *buff, size_t count, loff_t *pos)
 {
-        stampit();
+        ssize_t bytes_done;
+        u32 timing, page, base_flags, flags, ret;
+        u64 content;
+
+        if (!allow_writes)
+                return -EACCES;
 
         if (count % sizeof(u64))
                 return -EMSGSIZE;
@@ -123,20 +132,96 @@ static ssize_t bfin_otp_write(struct file *filp, const char __user *buff, size_t
         if (mutex_lock_interruptible(&bfin_otp_lock))
                 return -ERESTARTSYS;
 
-        /* need otp_init() documentation before this can be implemented */
+        stampit();
+
+        timing = bfin_otp_init_timing();
+        if (timing == 0) {
+                mutex_unlock(&bfin_otp_lock);
+                return -EIO;
+        }
+
+        base_flags = OTP_CHECK_FOR_PREV_WRITE;
+
+        bytes_done = 0;
+        page = *pos / (sizeof(u64) * 2);
+        while (bytes_done < count) {
+                flags = base_flags | (*pos % (sizeof(u64) * 2) ? OTP_UPPER_HALF : OTP_LOWER_HALF);
+                stamp("processing page %i (0x%x:%s) from %p", page, flags,
+                        (flags & OTP_UPPER_HALF ? "upper" : "lower"), buff + bytes_done);
+                if (copy_from_user(&content, buff + bytes_done, sizeof(content))) {
+                        bytes_done = -EFAULT;
+                        break;
+                }
+                ret = bfrom_OtpWrite(page, flags, &content);
+                if (ret & OTP_MASTER_ERROR) {
+                        stamp("error from otp: 0x%x", ret);
+                        bytes_done = -EIO;
+                        break;
+                }
+                if (flags & OTP_UPPER_HALF)
+                        ++page;
+                bytes_done += sizeof(content);
+                *pos += sizeof(content);
+        }
+
+        bfin_otp_deinit_timing(timing);
 
         mutex_unlock(&bfin_otp_lock);
 
+        return bytes_done;
+}
+
+static long bfin_otp_ioctl(struct file *filp, unsigned cmd, unsigned long arg)
+{
+        stampit();
+
+        switch (cmd) {
+        case OTPLOCK: {
+                u32 timing;
+                int ret = -EIO;
+
+                if (!allow_writes)
+                        return -EACCES;
+
+                if (mutex_lock_interruptible(&bfin_otp_lock))
+                        return -ERESTARTSYS;
+
+                timing = bfin_otp_init_timing();
+                if (timing) {
+                        u32 otp_result = bfrom_OtpWrite(arg, OTP_LOCK, NULL);
+                        stamp("locking page %lu resulted in 0x%x", arg, otp_result);
+                        if (!(otp_result & OTP_MASTER_ERROR))
+                                ret = 0;
+
+                        bfin_otp_deinit_timing(timing);
+                }
+
+                mutex_unlock(&bfin_otp_lock);
+
+                return ret;
+        }
+
+        case MEMLOCK:
+                allow_writes = false;
+                return 0;
+
+        case MEMUNLOCK:
+                allow_writes = true;
+                return 0;
+        }
+
         return -EINVAL;
 }
 #else
 # define bfin_otp_write NULL
+# define bfin_otp_ioctl NULL
 #endif
 
 static struct file_operations bfin_otp_fops = {
-        .owner    = THIS_MODULE,
-        .read     = bfin_otp_read,
-        .write    = bfin_otp_write,
+        .owner          = THIS_MODULE,
+        .unlocked_ioctl = bfin_otp_ioctl,
+        .read           = bfin_otp_read,
+        .write          = bfin_otp_write,
 };
 
 static struct miscdevice bfin_otp_misc_device = {
diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c
index 4a9f3492b921..70a770ac0138 100644
--- a/drivers/char/hpet.c
+++ b/drivers/char/hpet.c
@@ -166,9 +166,8 @@ static irqreturn_t hpet_interrupt(int irq, void *data)
                 unsigned long m, t;
 
                 t = devp->hd_ireqfreq;
-                m = read_counter(&devp->hd_hpet->hpet_mc);
-                write_counter(t + m + devp->hd_hpets->hp_delta,
-                              &devp->hd_timer->hpet_compare);
+                m = read_counter(&devp->hd_timer->hpet_compare);
+                write_counter(t + m, &devp->hd_timer->hpet_compare);
         }
 
         if (devp->hd_flags & HPET_SHARED_IRQ)
@@ -504,21 +503,25 @@ static int hpet_ioctl_ieon(struct hpet_dev *devp)
         g = v | Tn_32MODE_CNF_MASK | Tn_INT_ENB_CNF_MASK;
 
         if (devp->hd_flags & HPET_PERIODIC) {
-                write_counter(t, &timer->hpet_compare);
                 g |= Tn_TYPE_CNF_MASK;
-                v |= Tn_TYPE_CNF_MASK;
-                writeq(v, &timer->hpet_config);
-                v |= Tn_VAL_SET_CNF_MASK;
+                v |= Tn_TYPE_CNF_MASK | Tn_VAL_SET_CNF_MASK;
                 writeq(v, &timer->hpet_config);
                 local_irq_save(flags);
 
-                /* NOTE:  what we modify here is a hidden accumulator
+                /*
+                 * NOTE: First we modify the hidden accumulator
                  * register supported by periodic-capable comparators.
                  * We never want to modify the (single) counter; that
-                 * would affect all the comparators.
+                 * would affect all the comparators. The value written
+                 * is the counter value when the first interrupt is due.
                  */
                 m = read_counter(&hpet->hpet_mc);
                 write_counter(t + m + hpetp->hp_delta, &timer->hpet_compare);
+                /*
+                 * Then we modify the comparator, indicating the period
+                 * for subsequent interrupt.
+                 */
+                write_counter(t, &timer->hpet_compare);
         } else {
                 local_irq_save(flags);
                 m = read_counter(&hpet->hpet_mc);
diff --git a/drivers/char/hvc_console.c b/drivers/char/hvc_console.c
index 25ce15bb1c08..a632f25f144a 100644
--- a/drivers/char/hvc_console.c
+++ b/drivers/char/hvc_console.c
@@ -678,7 +678,7 @@ int hvc_poll(struct hvc_struct *hp)
 EXPORT_SYMBOL_GPL(hvc_poll);
 
 /**
- * hvc_resize() - Update terminal window size information.
+ * __hvc_resize() - Update terminal window size information.
  * @hp:         HVC console pointer
  * @ws:         Terminal window size structure
  *
@@ -687,12 +687,12 @@ EXPORT_SYMBOL_GPL(hvc_poll);
  *
  * Locking:     Locking free; the function MUST be called holding hp->lock
  */
-void hvc_resize(struct hvc_struct *hp, struct winsize ws)
+void __hvc_resize(struct hvc_struct *hp, struct winsize ws)
 {
         hp->ws = ws;
         schedule_work(&hp->tty_resize);
 }
-EXPORT_SYMBOL_GPL(hvc_resize);
+EXPORT_SYMBOL_GPL(__hvc_resize);
 
 /*
  * This kthread is either polling or interrupt driven.  This is determined by
diff --git a/drivers/char/hvc_console.h b/drivers/char/hvc_console.h
index 3c85d78c975c..10950ca706d8 100644
--- a/drivers/char/hvc_console.h
+++ b/drivers/char/hvc_console.h
@@ -28,6 +28,7 @@
 #define HVC_CONSOLE_H
 #include <linux/kref.h>
 #include <linux/tty.h>
+#include <linux/spinlock.h>
 
 /*
  * This is the max number of console adapters that can/will be found as
@@ -88,7 +89,16 @@ int hvc_poll(struct hvc_struct *hp);
 void hvc_kick(void);
 
 /* Resize hvc tty terminal window */
-extern void hvc_resize(struct hvc_struct *hp, struct winsize ws);
+extern void __hvc_resize(struct hvc_struct *hp, struct winsize ws);
+
+static inline void hvc_resize(struct hvc_struct *hp, struct winsize ws)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&hp->lock, flags);
+        __hvc_resize(hp, ws);
+        spin_unlock_irqrestore(&hp->lock, flags);
+}
 
 /* default notifier for irq based notification */
 extern int notifier_add_irq(struct hvc_struct *hp, int data);
diff --git a/drivers/char/hvc_iucv.c b/drivers/char/hvc_iucv.c
index 0ecac7e532f6..b8a5d654d3d0 100644
--- a/drivers/char/hvc_iucv.c
+++ b/drivers/char/hvc_iucv.c
@@ -273,7 +273,9 @@ static int hvc_iucv_write(struct hvc_iucv_private *priv,
         case MSG_TYPE_WINSIZE:
                 if (rb->mbuf->datalen != sizeof(struct winsize))
                         break;
-                hvc_resize(priv->hvc, *((struct winsize *) rb->mbuf->data));
+                /* The caller must ensure that the hvc is locked, which
+                 * is the case when called from hvc_iucv_get_chars() */
+                __hvc_resize(priv->hvc, *((struct winsize *) rb->mbuf->data));
                 break;
 
         case MSG_TYPE_ERROR:    /* ignored ... */
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 0aede1d6a9ea..6c8b65d069e5 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -690,7 +690,7 @@ static ssize_t read_zero(struct file * file, char __user * buf,
 
                 if (chunk > PAGE_SIZE)
                         chunk = PAGE_SIZE;      /* Just for latency reasons */
-                unwritten = clear_user(buf, chunk);
+                unwritten = __clear_user(buf, chunk);
                 written += chunk - unwritten;
                 if (unwritten)
                         break;
diff --git a/drivers/char/mwave/mwavedd.c b/drivers/char/mwave/mwavedd.c
index 94ad2c3bfc4a..a4ec50c95072 100644
--- a/drivers/char/mwave/mwavedd.c
+++ b/drivers/char/mwave/mwavedd.c
@@ -281,12 +281,6 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
                 case IOCTL_MW_REGISTER_IPC: {
                         unsigned int ipcnum = (unsigned int) ioarg;
         
-                        PRINTK_3(TRACE_MWAVE,
-                                "mwavedd::mwave_ioctl IOCTL_MW_REGISTER_IPC"
-                                " ipcnum %x entry usIntCount %x\n",
-                                ipcnum,
-                                pDrvData->IPCs[ipcnum].usIntCount);
-        
                         if (ipcnum >= ARRAY_SIZE(pDrvData->IPCs)) {
                                 PRINTK_ERROR(KERN_ERR_MWAVE
                                                 "mwavedd::mwave_ioctl:"
@@ -295,6 +289,12 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
                                                 ipcnum);
                                 return -EINVAL;
                         }
+                        PRINTK_3(TRACE_MWAVE,
+                                "mwavedd::mwave_ioctl IOCTL_MW_REGISTER_IPC"
+                                " ipcnum %x entry usIntCount %x\n",
+                                ipcnum,
+                                pDrvData->IPCs[ipcnum].usIntCount);
+
                         lock_kernel();
                         pDrvData->IPCs[ipcnum].bIsHere = FALSE;
                         pDrvData->IPCs[ipcnum].bIsEnabled = TRUE;
@@ -310,11 +310,6 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
                 case IOCTL_MW_GET_IPC: {
                         unsigned int ipcnum = (unsigned int) ioarg;
         
-                        PRINTK_3(TRACE_MWAVE,
-                                "mwavedd::mwave_ioctl IOCTL_MW_GET_IPC"
-                                " ipcnum %x, usIntCount %x\n",
-                                ipcnum,
-                                pDrvData->IPCs[ipcnum].usIntCount);
                         if (ipcnum >= ARRAY_SIZE(pDrvData->IPCs)) {
                                 PRINTK_ERROR(KERN_ERR_MWAVE
                                                 "mwavedd::mwave_ioctl:"
@@ -322,6 +317,11 @@ static long mwave_ioctl(struct file *file, unsigned int iocmd,
                                                 " Invalid ipcnum %x\n", ipcnum);
                                 return -EINVAL;
                         }
+                        PRINTK_3(TRACE_MWAVE,
+                                "mwavedd::mwave_ioctl IOCTL_MW_GET_IPC"
+                                " ipcnum %x, usIntCount %x\n",
+                                ipcnum,
+                                pDrvData->IPCs[ipcnum].usIntCount);
         
                         lock_kernel();
                         if (pDrvData->IPCs[ipcnum].bIsEnabled == TRUE) {
diff --git a/drivers/char/random.c b/drivers/char/random.c
index d8a9255e1a3f..04b505e5a5e2 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -1231,7 +1231,7 @@ static char sysctl_bootid[16];
  * as an ASCII string in the standard UUID format.  If accesses via the
  * sysctl system call, it is returned as 16 bytes of binary data.
  */
-static int proc_do_uuid(ctl_table *table, int write, struct file *filp,
+static int proc_do_uuid(ctl_table *table, int write,
                         void __user *buffer, size_t *lenp, loff_t *ppos)
 {
         ctl_table fake_table;
@@ -1254,7 +1254,7 @@ static int proc_do_uuid(ctl_table *table, int write, struct file *filp,
         fake_table.data = buf;
         fake_table.maxlen = sizeof(buf);
 
-        return proc_dostring(&fake_table, write, filp, buffer, lenp, ppos);
+        return proc_dostring(&fake_table, write, buffer, lenp, ppos);
 }
 
 static int uuid_strategy(ctl_table *table,
diff --git a/drivers/char/rio/rioctrl.c b/drivers/char/rio/rioctrl.c
index eecee0f576d2..74339559f0b9 100644
--- a/drivers/char/rio/rioctrl.c
+++ b/drivers/char/rio/rioctrl.c
@@ -873,7 +873,7 @@ int riocontrol(struct rio_info *p, dev_t dev, int cmd, unsigned long arg, int su
                 /*
                  ** It is important that the product code is an unsigned object!
                  */
-                if (DownLoad.ProductCode > MAX_PRODUCT) {
+                if (DownLoad.ProductCode >= MAX_PRODUCT) {
                         rio_dprintk(RIO_DEBUG_CTRL, "RIO_DOWNLOAD: Bad product code %d passed\n", DownLoad.ProductCode);
                         p->RIOError.Error = NO_SUCH_PRODUCT;
                         return -ENXIO;
diff --git a/drivers/char/uv_mmtimer.c b/drivers/char/uv_mmtimer.c
new file mode 100644
index 000000000000..867b67be9f0a
--- /dev/null
+++ b/drivers/char/uv_mmtimer.c
@@ -0,0 +1,216 @@
+/*
+ * Timer device implementation for SGI UV platform.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (c) 2009 Silicon Graphics, Inc.  All rights reserved.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/ioctl.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/mm.h>
+#include <linux/fs.h>
+#include <linux/mmtimer.h>
+#include <linux/miscdevice.h>
+#include <linux/posix-timers.h>
+#include <linux/interrupt.h>
+#include <linux/time.h>
+#include <linux/math64.h>
+#include <linux/smp_lock.h>
+
+#include <asm/genapic.h>
+#include <asm/uv/uv_hub.h>
+#include <asm/uv/bios.h>
+#include <asm/uv/uv.h>
+
+MODULE_AUTHOR("Dimitri Sivanich <sivanich@sgi.com>");
+MODULE_DESCRIPTION("SGI UV Memory Mapped RTC Timer");
+MODULE_LICENSE("GPL");
+
+/* name of the device, usually in /dev */
+#define UV_MMTIMER_NAME "mmtimer"
+#define UV_MMTIMER_DESC "SGI UV Memory Mapped RTC Timer"
+#define UV_MMTIMER_VERSION "1.0"
+
+static long uv_mmtimer_ioctl(struct file *file, unsigned int cmd,
+                                                unsigned long arg);
+static int uv_mmtimer_mmap(struct file *file, struct vm_area_struct *vma);
+
+/*
+ * Period in femtoseconds (10^-15 s)
+ */
+static unsigned long uv_mmtimer_femtoperiod;
+
+static const struct file_operations uv_mmtimer_fops = {
+        .owner = THIS_MODULE,
+        .mmap = uv_mmtimer_mmap,
+        .unlocked_ioctl = uv_mmtimer_ioctl,
+};
+
+/**
+ * uv_mmtimer_ioctl - ioctl interface for /dev/uv_mmtimer
+ * @file: file structure for the device
+ * @cmd: command to execute
+ * @arg: optional argument to command
+ *
+ * Executes the command specified by @cmd.  Returns 0 for success, < 0 for
+ * failure.
+ *
+ * Valid commands:
+ *
+ * %MMTIMER_GETOFFSET - Should return the offset (relative to the start
+ * of the page where the registers are mapped) for the counter in question.
+ *
+ * %MMTIMER_GETRES - Returns the resolution of the clock in femto (10^-15)
+ * seconds
+ *
+ * %MMTIMER_GETFREQ - Copies the frequency of the clock in Hz to the address
+ * specified by @arg
+ *
+ * %MMTIMER_GETBITS - Returns the number of bits in the clock's counter
+ *
+ * %MMTIMER_MMAPAVAIL - Returns 1 if registers can be mmap'd into userspace
+ *
+ * %MMTIMER_GETCOUNTER - Gets the current value in the counter and places it
+ * in the address specified by @arg.
+ */
+static long uv_mmtimer_ioctl(struct file *file, unsigned int cmd,
+                                                unsigned long arg)
+{
+        int ret = 0;
+
+        switch (cmd) {
+        case MMTIMER_GETOFFSET: /* offset of the counter */
+                /*
+                 * UV RTC register is on its own page
+                 */
+                if (PAGE_SIZE <= (1 << 16))
+                        ret = ((UV_LOCAL_MMR_BASE | UVH_RTC) & (PAGE_SIZE-1))
+                                / 8;
+                else
+                        ret = -ENOSYS;
+                break;
+
+        case MMTIMER_GETRES: /* resolution of the clock in 10^-15 s */
+                if (copy_to_user((unsigned long __user *)arg,
+                                &uv_mmtimer_femtoperiod, sizeof(unsigned long)))
+                        ret = -EFAULT;
+                break;
+
+        case MMTIMER_GETFREQ: /* frequency in Hz */
+                if (copy_to_user((unsigned long __user *)arg,
+                                &sn_rtc_cycles_per_second,
+                                sizeof(unsigned long)))
+                        ret = -EFAULT;
+                break;
+
+        case MMTIMER_GETBITS: /* number of bits in the clock */
+                ret = hweight64(UVH_RTC_REAL_TIME_CLOCK_MASK);
+                break;
+
+        case MMTIMER_MMAPAVAIL: /* can we mmap the clock into userspace? */
+                ret = (PAGE_SIZE <= (1 << 16)) ? 1 : 0;
+                break;
+
+        case MMTIMER_GETCOUNTER:
+                if (copy_to_user((unsigned long __user *)arg,
+                                (unsigned long *)uv_local_mmr_address(UVH_RTC),
+                                sizeof(unsigned long)))
+                        ret = -EFAULT;
+                break;
+        default:
+                ret = -ENOTTY;
+                break;
+        }
+        return ret;
+}
+
+/**
+ * uv_mmtimer_mmap - maps the clock's registers into userspace
+ * @file: file structure for the device
+ * @vma: VMA to map the registers into
+ *
+ * Calls remap_pfn_range() to map the clock's registers into
+ * the calling process' address space.
+ */
+static int uv_mmtimer_mmap(struct file *file, struct vm_area_struct *vma)
+{
+        unsigned long uv_mmtimer_addr;
+
+        if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+                return -EINVAL;
+
+        if (vma->vm_flags & VM_WRITE)
+                return -EPERM;
+
+        if (PAGE_SIZE > (1 << 16))
+                return -ENOSYS;
+
+        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+        uv_mmtimer_addr = UV_LOCAL_MMR_BASE | UVH_RTC;
+        uv_mmtimer_addr &= ~(PAGE_SIZE - 1);
+        uv_mmtimer_addr &= 0xfffffffffffffffUL;
+
+        if (remap_pfn_range(vma, vma->vm_start, uv_mmtimer_addr >> PAGE_SHIFT,
+                                        PAGE_SIZE, vma->vm_page_prot)) {
+                printk(KERN_ERR "remap_pfn_range failed in uv_mmtimer_mmap\n");
+                return -EAGAIN;
+        }
+
+        return 0;
+}
+
+static struct miscdevice uv_mmtimer_miscdev = {
+        MISC_DYNAMIC_MINOR,
+        UV_MMTIMER_NAME,
+        &uv_mmtimer_fops
+};
+
+
+/**
+ * uv_mmtimer_init - device initialization routine
+ *
+ * Does initial setup for the uv_mmtimer device.
+ */
+static int __init uv_mmtimer_init(void)
+{
+        if (!is_uv_system()) {
+                printk(KERN_ERR "%s: Hardware unsupported\n", UV_MMTIMER_NAME);
+                return -1;
+        }
+
+        /*
+         * Sanity check the cycles/sec variable
+         */
+        if (sn_rtc_cycles_per_second < 100000) {
+                printk(KERN_ERR "%s: unable to determine clock frequency\n",
+                       UV_MMTIMER_NAME);
+                return -1;
+        }
+
+        uv_mmtimer_femtoperiod = ((unsigned long)1E15 +
+                                sn_rtc_cycles_per_second / 2) /
+                                sn_rtc_cycles_per_second;
+
+        if (misc_register(&uv_mmtimer_miscdev)) {
+                printk(KERN_ERR "%s: failed to register device\n",
+                       UV_MMTIMER_NAME);
+                return -1;
+        }
+
+        printk(KERN_INFO "%s: v%s, %ld MHz\n", UV_MMTIMER_DESC,
+                UV_MMTIMER_VERSION,
+                sn_rtc_cycles_per_second/(unsigned long)1E6);
+
+        return 0;
+}
+
+module_init(uv_mmtimer_init);
diff --git a/drivers/dca/dca-core.c b/drivers/dca/dca-core.c
index 25b743abfb59..52e6bb70a490 100644
--- a/drivers/dca/dca-core.c
+++ b/drivers/dca/dca-core.c
@@ -28,7 +28,7 @@
 #include <linux/device.h>
 #include <linux/dca.h>
 
-#define DCA_VERSION "1.8"
+#define DCA_VERSION "1.12.1"
 
 MODULE_VERSION(DCA_VERSION);
 MODULE_LICENSE("GPL");
@@ -36,20 +36,92 @@ MODULE_AUTHOR("Intel Corporation");
 
 static DEFINE_SPINLOCK(dca_lock);
 
-static LIST_HEAD(dca_providers);
+static LIST_HEAD(dca_domains);
 
-static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
+static struct pci_bus *dca_pci_rc_from_dev(struct device *dev)
 {
-        struct dca_provider *dca, *ret = NULL;
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct pci_bus *bus = pdev->bus;
 
-        list_for_each_entry(dca, &dca_providers, node) {
-                if ((!dev) || (dca->ops->dev_managed(dca, dev))) {
-                        ret = dca;
-                        break;
-                }
+        while (bus->parent)
+                bus = bus->parent;
+
+        return bus;
+}
+
+static struct dca_domain *dca_allocate_domain(struct pci_bus *rc)
+{
+        struct dca_domain *domain;
+
+        domain = kzalloc(sizeof(*domain), GFP_NOWAIT);
+        if (!domain)
+                return NULL;
+
+        INIT_LIST_HEAD(&domain->dca_providers);
+        domain->pci_rc = rc;
+
+        return domain;
+}
+
+static void dca_free_domain(struct dca_domain *domain)
+{
+        list_del(&domain->node);
+        kfree(domain);
+}
+
+static struct dca_domain *dca_find_domain(struct pci_bus *rc)
+{
+        struct dca_domain *domain;
+
+        list_for_each_entry(domain, &dca_domains, node)
+                if (domain->pci_rc == rc)
+                        return domain;
+
+        return NULL;
+}
+
+static struct dca_domain *dca_get_domain(struct device *dev)
+{
+        struct pci_bus *rc;
+        struct dca_domain *domain;
+
+        rc = dca_pci_rc_from_dev(dev);
+        domain = dca_find_domain(rc);
+
+        if (!domain) {
+                domain = dca_allocate_domain(rc);
+                if (domain)
+                        list_add(&domain->node, &dca_domains);
+        }
+
+        return domain;
+}
+
+static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
+{
+        struct dca_provider *dca;
+        struct pci_bus *rc;
+        struct dca_domain *domain;
+
+        if (dev) {
+                rc = dca_pci_rc_from_dev(dev);
+                domain = dca_find_domain(rc);
+                if (!domain)
+                        return NULL;
+        } else {
+                if (!list_empty(&dca_domains))
+                        domain = list_first_entry(&dca_domains,
+                                                  struct dca_domain,
+                                                  node);
+                else
+                        return NULL;
         }
 
-        return ret;
+        list_for_each_entry(dca, &domain->dca_providers, node)
+                if ((!dev) || (dca->ops->dev_managed(dca, dev)))
+                        return dca;
+
+        return NULL;
 }
 
 /**
@@ -61,6 +133,8 @@ int dca_add_requester(struct device *dev)
         struct dca_provider *dca;
         int err, slot = -ENODEV;
         unsigned long flags;
+        struct pci_bus *pci_rc;
+        struct dca_domain *domain;
 
         if (!dev)
                 return -EFAULT;
@@ -74,7 +148,14 @@ int dca_add_requester(struct device *dev)
                 return -EEXIST;
         }
 
-        list_for_each_entry(dca, &dca_providers, node) {
+        pci_rc = dca_pci_rc_from_dev(dev);
+        domain = dca_find_domain(pci_rc);
+        if (!domain) {
+                spin_unlock_irqrestore(&dca_lock, flags);
+                return -ENODEV;
+        }
+
+        list_for_each_entry(dca, &domain->dca_providers, node) {
                 slot = dca->ops->add_requester(dca, dev);
                 if (slot >= 0)
                         break;
@@ -222,13 +303,19 @@ int register_dca_provider(struct dca_provider *dca, struct device *dev)
 {
         int err;
         unsigned long flags;
+        struct dca_domain *domain;
 
         err = dca_sysfs_add_provider(dca, dev);
         if (err)
                 return err;
 
         spin_lock_irqsave(&dca_lock, flags);
-        list_add(&dca->node, &dca_providers);
+        domain = dca_get_domain(dev);
+        if (!domain) {
+                spin_unlock_irqrestore(&dca_lock, flags);
+                return -ENODEV;
+        }
+        list_add(&dca->node, &domain->dca_providers);
         spin_unlock_irqrestore(&dca_lock, flags);
 
         blocking_notifier_call_chain(&dca_provider_chain,
@@ -241,15 +328,24 @@ EXPORT_SYMBOL_GPL(register_dca_provider);
  * unregister_dca_provider - remove a dca provider
  * @dca - struct created by alloc_dca_provider()
  */
-void unregister_dca_provider(struct dca_provider *dca)
+void unregister_dca_provider(struct dca_provider *dca, struct device *dev)
 {
         unsigned long flags;
+        struct pci_bus *pci_rc;
+        struct dca_domain *domain;
 
         blocking_notifier_call_chain(&dca_provider_chain,
                                      DCA_PROVIDER_REMOVE, NULL);
 
         spin_lock_irqsave(&dca_lock, flags);
+
         list_del(&dca->node);
+
+        pci_rc = dca_pci_rc_from_dev(dev);
+        domain = dca_find_domain(pci_rc);
+        if (list_empty(&domain->dca_providers))
+                dca_free_domain(domain);
+
         spin_unlock_irqrestore(&dca_lock, flags);
 
         dca_sysfs_remove_provider(dca);
@@ -276,7 +372,7 @@ EXPORT_SYMBOL_GPL(dca_unregister_notify);
 
 static int __init dca_init(void)
 {
-        printk(KERN_ERR "dca service started, version %s\n", DCA_VERSION);
+        pr_info("dca service started, version %s\n", DCA_VERSION);
         return dca_sysfs_init();
 }
 
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 81e1020fb514..5903a88351bf 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -17,11 +17,15 @@ if DMADEVICES
 
 comment "DMA Devices"
 
+config ASYNC_TX_DISABLE_CHANNEL_SWITCH
+        bool
+
 config INTEL_IOATDMA
         tristate "Intel I/OAT DMA support"
         depends on PCI && X86
         select DMA_ENGINE
         select DCA
+        select ASYNC_TX_DISABLE_CHANNEL_SWITCH
         help
           Enable support for the Intel(R) I/OAT DMA engine present
           in recent Intel Xeon chipsets.
@@ -97,6 +101,14 @@ config TXX9_DMAC
           Support the TXx9 SoC internal DMA controller.  This can be
           integrated in chips such as the Toshiba TX4927/38/39.
 
+config SH_DMAE
+        tristate "Renesas SuperH DMAC support"
+        depends on SUPERH && SH_DMA
+        depends on !SH_DMA_API
+        select DMA_ENGINE
+        help
+          Enable support for the Renesas SuperH DMA controllers.
+
 config DMA_ENGINE
         bool
 
@@ -116,7 +128,7 @@ config NET_DMA
 
 config ASYNC_TX_DMA
         bool "Async_tx: Offload support for the async_tx api"
-        depends on DMA_ENGINE && !HIGHMEM64G
+        depends on DMA_ENGINE
         help
           This allows the async_tx api to take advantage of offload engines for
           memcpy, memset, xor, and raid6 p+q operations.  If your platform has
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 40e1e0083571..eca71ba78ae9 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,8 +1,7 @@
 obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
 obj-$(CONFIG_NET_DMA) += iovlock.o
 obj-$(CONFIG_DMATEST) += dmatest.o
-obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
-ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
+obj-$(CONFIG_INTEL_IOATDMA) += ioat/
 obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
 obj-$(CONFIG_FSL_DMA) += fsldma.o
 obj-$(CONFIG_MV_XOR) += mv_xor.o
@@ -10,3 +9,4 @@ obj-$(CONFIG_DW_DMAC) += dw_dmac.o
 obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
 obj-$(CONFIG_MX3_IPU) += ipu/
 obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
+obj-$(CONFIG_SH_DMAE) += shdma.o
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index c8522e6f1ad2..7585c4164bd5 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -87,6 +87,7 @@ static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
         desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
         if (desc) {
                 memset(desc, 0, sizeof(struct at_desc));
+                INIT_LIST_HEAD(&desc->tx_list);
                 dma_async_tx_descriptor_init(&desc->txd, chan);
                 /* txd.flags will be overwritten in prep functions */
                 desc->txd.flags = DMA_CTRL_ACK;
@@ -150,11 +151,11 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
                 struct at_desc *child;
 
                 spin_lock_bh(&atchan->lock);
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         dev_vdbg(chan2dev(&atchan->chan_common),
                                         "moving child desc %p to freelist\n",
                                         child);
-                list_splice_init(&desc->txd.tx_list, &atchan->free_list);
+                list_splice_init(&desc->tx_list, &atchan->free_list);
                 dev_vdbg(chan2dev(&atchan->chan_common),
                          "moving desc %p to freelist\n", desc);
                 list_add(&desc->desc_node, &atchan->free_list);
@@ -247,30 +248,33 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
         param = txd->callback_param;
 
         /* move children to free_list */
-        list_splice_init(&txd->tx_list, &atchan->free_list);
+        list_splice_init(&desc->tx_list, &atchan->free_list);
         /* move myself to free_list */
         list_move(&desc->desc_node, &atchan->free_list);
 
         /* unmap dma addresses */
-        if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-                if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
-                        dma_unmap_single(chan2parent(&atchan->chan_common),
-                                        desc->lli.daddr,
-                                        desc->len, DMA_FROM_DEVICE);
-                else
-                        dma_unmap_page(chan2parent(&atchan->chan_common),
-                                        desc->lli.daddr,
-                                        desc->len, DMA_FROM_DEVICE);
-        }
-        if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-                if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
-                        dma_unmap_single(chan2parent(&atchan->chan_common),
-                                        desc->lli.saddr,
-                                        desc->len, DMA_TO_DEVICE);
-                else
-                        dma_unmap_page(chan2parent(&atchan->chan_common),
-                                        desc->lli.saddr,
-                                        desc->len, DMA_TO_DEVICE);
+        if (!atchan->chan_common.private) {
+                struct device *parent = chan2parent(&atchan->chan_common);
+                if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                                dma_unmap_single(parent,
+                                                desc->lli.daddr,
+                                                desc->len, DMA_FROM_DEVICE);
+                        else
+                                dma_unmap_page(parent,
+                                                desc->lli.daddr,
+                                                desc->len, DMA_FROM_DEVICE);
+                }
+                if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                                dma_unmap_single(parent,
+                                                desc->lli.saddr,
+                                                desc->len, DMA_TO_DEVICE);
+                        else
+                                dma_unmap_page(parent,
+                                                desc->lli.saddr,
+                                                desc->len, DMA_TO_DEVICE);
+                }
         }
 
         /*
@@ -334,7 +338,7 @@ static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
                         /* This one is currently in progress */
                         return;
 
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         if (!(child->lli.ctrla & ATC_DONE))
                                 /* Currently in progress */
                                 return;
@@ -407,7 +411,7 @@ static void atc_handle_error(struct at_dma_chan *atchan)
         dev_crit(chan2dev(&atchan->chan_common),
                         "  cookie: %d\n", bad_desc->txd.cookie);
         atc_dump_lli(atchan, &bad_desc->lli);
-        list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                 atc_dump_lli(atchan, &child->lli);
 
         /* Pretend the descriptor completed successfully */
@@ -587,7 +591,7 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                         prev->lli.dscr = desc->txd.phys;
                         /* insert the link descriptor to the LD ring */
                         list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                                        &first->tx_list);
                 }
                 prev = desc;
         }
@@ -646,8 +650,6 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 
         reg_width = atslave->reg_width;
 
-        sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
         ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
         ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
 
@@ -687,7 +689,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                 prev->lli.dscr = desc->txd.phys;
                                 /* insert the link descriptor to the LD ring */
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
@@ -729,7 +731,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                 prev->lli.dscr = desc->txd.phys;
                                 /* insert the link descriptor to the LD ring */
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h
index 4c972afc49ec..495457e3dc4b 100644
--- a/drivers/dma/at_hdmac_regs.h
+++ b/drivers/dma/at_hdmac_regs.h
@@ -165,6 +165,7 @@ struct at_desc {
         struct at_lli                   lli;
 
         /* THEN values for driver housekeeping */
+        struct list_head                tx_list;
         struct dma_async_tx_descriptor  txd;
         struct list_head                desc_node;
         size_t                          len;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 5a87384ea4ff..bd0b248de2cf 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -608,6 +608,40 @@ void dmaengine_put(void)
 }
 EXPORT_SYMBOL(dmaengine_put);
 
+static bool device_has_all_tx_types(struct dma_device *device)
+{
+        /* A device that satisfies this test has channels that will never cause
+         * an async_tx channel switch event as all possible operation types can
+         * be handled.
+         */
+        #ifdef CONFIG_ASYNC_TX_DMA
+        if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE)
+        if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_MEMSET) || defined(CONFIG_ASYNC_MEMSET_MODULE)
+        if (!dma_has_cap(DMA_MEMSET, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
+        if (!dma_has_cap(DMA_XOR, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
+        if (!dma_has_cap(DMA_PQ, device->cap_mask))
+                return false;
+        #endif
+
+        return true;
+}
+
 static int get_dma_id(struct dma_device *device)
 {
         int rc;
@@ -644,8 +678,12 @@ int dma_async_device_register(struct dma_device *device)
                 !device->device_prep_dma_memcpy);
         BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
                 !device->device_prep_dma_xor);
-        BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
-                !device->device_prep_dma_zero_sum);
+        BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) &&
+                !device->device_prep_dma_xor_val);
+        BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) &&
+                !device->device_prep_dma_pq);
+        BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) &&
+                !device->device_prep_dma_pq_val);
         BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
                 !device->device_prep_dma_memset);
         BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
@@ -661,6 +699,12 @@ int dma_async_device_register(struct dma_device *device)
         BUG_ON(!device->device_issue_pending);
         BUG_ON(!device->dev);
 
+        /* note: this only matters in the
+         * CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH=y case
+         */
+        if (device_has_all_tx_types(device))
+                dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
+
         idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
         if (!idr_ref)
                 return -ENOMEM;
@@ -933,55 +977,29 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
 {
         tx->chan = chan;
         spin_lock_init(&tx->lock);
-        INIT_LIST_HEAD(&tx->tx_list);
 }
 EXPORT_SYMBOL(dma_async_tx_descriptor_init);
 
 /* dma_wait_for_async_tx - spin wait for a transaction to complete
  * @tx: in-flight transaction to wait on
- *
- * This routine assumes that tx was obtained from a call to async_memcpy,
- * async_xor, async_memset, etc which ensures that tx is "in-flight" (prepped
- * and submitted).  Walking the parent chain is only meant to cover for DMA
- * drivers that do not implement the DMA_INTERRUPT capability and may race with
- * the driver's descriptor cleanup routine.
  */
 enum dma_status
 dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
 {
-        enum dma_status status;
-        struct dma_async_tx_descriptor *iter;
-        struct dma_async_tx_descriptor *parent;
+        unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
 
         if (!tx)
                 return DMA_SUCCESS;
 
-        WARN_ONCE(tx->parent, "%s: speculatively walking dependency chain for"
-                  " %s\n", __func__, dma_chan_name(tx->chan));
-
-        /* poll through the dependency chain, return when tx is complete */
-        do {
-                iter = tx;
-
-                /* find the root of the unsubmitted dependency chain */
-                do {
-                        parent = iter->parent;
-                        if (!parent)
-                                break;
-                        else
-                                iter = parent;
-                } while (parent);
-
-                /* there is a small window for ->parent == NULL and
-                 * ->cookie == -EBUSY
-                 */
-                while (iter->cookie == -EBUSY)
-                        cpu_relax();
-
-                status = dma_sync_wait(iter->chan, iter->cookie);
-        } while (status == DMA_IN_PROGRESS || (iter != tx));
-
-        return status;
+        while (tx->cookie == -EBUSY) {
+                if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
+                        pr_err("%s timeout waiting for descriptor submission\n",
+                                __func__);
+                        return DMA_ERROR;
+                }
+                cpu_relax();
+        }
+        return dma_sync_wait(tx->chan, tx->cookie);
 }
 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
 
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index d93017fc7872..a32a4cf7b1e0 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -48,6 +48,11 @@ module_param(xor_sources, uint, S_IRUGO);
 MODULE_PARM_DESC(xor_sources,
                 "Number of xor source buffers (default: 3)");
 
+static unsigned int pq_sources = 3;
+module_param(pq_sources, uint, S_IRUGO);
+MODULE_PARM_DESC(pq_sources,
+                "Number of p+q source buffers (default: 3)");
+
 /*
  * Initialization patterns. All bytes in the source buffer has bit 7
  * set, all bytes in the destination buffer has bit 7 cleared.
@@ -232,6 +237,7 @@ static int dmatest_func(void *data)
         dma_cookie_t            cookie;
         enum dma_status         status;
         enum dma_ctrl_flags     flags;
+        u8                      pq_coefs[pq_sources];
         int                     ret;
         int                     src_cnt;
         int                     dst_cnt;
@@ -248,6 +254,11 @@ static int dmatest_func(void *data)
         else if (thread->type == DMA_XOR) {
                 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */
                 dst_cnt = 1;
+        } else if (thread->type == DMA_PQ) {
+                src_cnt = pq_sources | 1; /* force odd to ensure dst = src */
+                dst_cnt = 2;
+                for (i = 0; i < pq_sources; i++)
+                        pq_coefs[i] = 1;
         } else
                 goto err_srcs;
 
@@ -283,6 +294,7 @@ static int dmatest_func(void *data)
                 dma_addr_t dma_dsts[dst_cnt];
                 struct completion cmp;
                 unsigned long tmo = msecs_to_jiffies(3000);
+                u8 align = 0;
 
                 total_tests++;
 
@@ -290,6 +302,18 @@ static int dmatest_func(void *data)
                 src_off = dmatest_random() % (test_buf_size - len + 1);
                 dst_off = dmatest_random() % (test_buf_size - len + 1);
 
+                /* honor alignment restrictions */
+                if (thread->type == DMA_MEMCPY)
+                        align = dev->copy_align;
+                else if (thread->type == DMA_XOR)
+                        align = dev->xor_align;
+                else if (thread->type == DMA_PQ)
+                        align = dev->pq_align;
+
+                len = (len >> align) << align;
+                src_off = (src_off >> align) << align;
+                dst_off = (dst_off >> align) << align;
+
                 dmatest_init_srcs(thread->srcs, src_off, len);
                 dmatest_init_dsts(thread->dsts, dst_off, len);
 
@@ -306,6 +330,7 @@ static int dmatest_func(void *data)
                                                      DMA_BIDIRECTIONAL);
                 }
 
+
                 if (thread->type == DMA_MEMCPY)
                         tx = dev->device_prep_dma_memcpy(chan,
                                                          dma_dsts[0] + dst_off,
@@ -316,6 +341,15 @@ static int dmatest_func(void *data)
                                                       dma_dsts[0] + dst_off,
                                                       dma_srcs, xor_sources,
                                                       len, flags);
+                else if (thread->type == DMA_PQ) {
+                        dma_addr_t dma_pq[dst_cnt];
+
+                        for (i = 0; i < dst_cnt; i++)
+                                dma_pq[i] = dma_dsts[i] + dst_off;
+                        tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
+                                                     pq_sources, pq_coefs,
+                                                     len, flags);
+                }
 
                 if (!tx) {
                         for (i = 0; i < src_cnt; i++)
@@ -459,6 +493,8 @@ static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_ty
                 op = "copy";
         else if (type == DMA_XOR)
                 op = "xor";
+        else if (type == DMA_PQ)
+                op = "pq";
         else
                 return -EINVAL;
 
@@ -514,6 +550,10 @@ static int dmatest_add_channel(struct dma_chan *chan)
                 cnt = dmatest_add_threads(dtc, DMA_XOR);
                 thread_count += cnt > 0 ? cnt : 0;
         }
+        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
+                cnt = dmatest_add_threads(dtc, DMA_PQ);
+                thread_count += cnt > 0 ?: 0;
+        }
 
         pr_info("dmatest: Started %u threads using %s\n",
                 thread_count, dma_chan_name(chan));
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
index 933c143b6a74..2eea823516a7 100644
--- a/drivers/dma/dw_dmac.c
+++ b/drivers/dma/dw_dmac.c
@@ -116,7 +116,7 @@ static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
         struct dw_desc  *child;
 
-        list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &desc->tx_list, desc_node)
                 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
                                 child->txd.phys, sizeof(child->lli),
                                 DMA_TO_DEVICE);
@@ -137,11 +137,11 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
                 dwc_sync_desc_for_cpu(dwc, desc);
 
                 spin_lock_bh(&dwc->lock);
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         dev_vdbg(chan2dev(&dwc->chan),
                                         "moving child desc %p to freelist\n",
                                         child);
-                list_splice_init(&desc->txd.tx_list, &dwc->free_list);
+                list_splice_init(&desc->tx_list, &dwc->free_list);
                 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
                 list_add(&desc->desc_node, &dwc->free_list);
                 spin_unlock_bh(&dwc->lock);
@@ -209,19 +209,28 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
         param = txd->callback_param;
 
         dwc_sync_desc_for_cpu(dwc, desc);
-        list_splice_init(&txd->tx_list, &dwc->free_list);
+        list_splice_init(&desc->tx_list, &dwc->free_list);
         list_move(&desc->desc_node, &dwc->free_list);
 
-        /*
-         * We use dma_unmap_page() regardless of how the buffers were
-         * mapped before they were submitted...
-         */
-        if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP))
-                dma_unmap_page(chan2parent(&dwc->chan), desc->lli.dar,
-                               desc->len, DMA_FROM_DEVICE);
-        if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP))
-                dma_unmap_page(chan2parent(&dwc->chan), desc->lli.sar,
-                               desc->len, DMA_TO_DEVICE);
+        if (!dwc->chan.private) {
+                struct device *parent = chan2parent(&dwc->chan);
+                if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                                dma_unmap_single(parent, desc->lli.dar,
+                                                desc->len, DMA_FROM_DEVICE);
+                        else
+                                dma_unmap_page(parent, desc->lli.dar,
+                                                desc->len, DMA_FROM_DEVICE);
+                }
+                if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                                dma_unmap_single(parent, desc->lli.sar,
+                                                desc->len, DMA_TO_DEVICE);
+                        else
+                                dma_unmap_page(parent, desc->lli.sar,
+                                                desc->len, DMA_TO_DEVICE);
+                }
+        }
 
         /*
          * The API requires that no submissions are done from a
@@ -289,7 +298,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
                         /* This one is currently in progress */
                         return;
 
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         if (child->lli.llp == llp)
                                 /* Currently in progress */
                                 return;
@@ -356,7 +365,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
         dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
                         "  cookie: %d\n", bad_desc->txd.cookie);
         dwc_dump_lli(dwc, &bad_desc->lli);
-        list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                 dwc_dump_lli(dwc, &child->lli);
 
         /* Pretend the descriptor completed successfully */
@@ -608,7 +617,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                                         prev->txd.phys, sizeof(prev->lli),
                                         DMA_TO_DEVICE);
                         list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                                        &first->tx_list);
                 }
                 prev = desc;
         }
@@ -658,8 +667,6 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
         reg_width = dws->reg_width;
         prev = first = NULL;
 
-        sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
         switch (direction) {
         case DMA_TO_DEVICE:
                 ctllo = (DWC_DEFAULT_CTLLO
@@ -700,7 +707,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                                 sizeof(prev->lli),
                                                 DMA_TO_DEVICE);
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
@@ -746,7 +753,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                                 sizeof(prev->lli),
                                                 DMA_TO_DEVICE);
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
@@ -902,6 +909,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
                         break;
                 }
 
+                INIT_LIST_HEAD(&desc->tx_list);
                 dma_async_tx_descriptor_init(&desc->txd, chan);
                 desc->txd.tx_submit = dwc_tx_submit;
                 desc->txd.flags = DMA_CTRL_ACK;
diff --git a/drivers/dma/dw_dmac_regs.h b/drivers/dma/dw_dmac_regs.h
index 13a580767031..d9a939f67f46 100644
--- a/drivers/dma/dw_dmac_regs.h
+++ b/drivers/dma/dw_dmac_regs.h
@@ -217,6 +217,7 @@ struct dw_desc {
 
         /* THEN values for driver housekeeping */
         struct list_head                desc_node;
+        struct list_head                tx_list;
         struct dma_async_tx_descriptor  txd;
         size_t                          len;
 };
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index ef87a8984145..296f9e747fac 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -34,6 +34,7 @@
 #include <linux/dmapool.h>
 #include <linux/of_platform.h>
 
+#include <asm/fsldma.h>
 #include "fsldma.h"
 
 static void dma_init(struct fsl_dma_chan *fsl_chan)
@@ -280,28 +281,40 @@ static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size)
 }
 
 /**
- * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * fsl_chan_set_request_count - Set DMA Request Count for external control
  * @fsl_chan : Freescale DMA channel
- * @size     : Pause control size, 0 for disable external pause control.
- *             The maximum is 1024.
+ * @size     : Number of bytes to transfer in a single request
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA request count is how many bytes are allowed to transfer before
+ * pausing the channel, after which a new assertion of DREQ# resumes channel
+ * operation.
  *
- * The Freescale DMA channel can be controlled by the external
- * signal DREQ#. The pause control size is how many bytes are allowed
- * to transfer before pausing the channel, after which a new assertion
- * of DREQ# resumes channel operation.
+ * A size of 0 disables external pause control. The maximum size is 1024.
  */
-static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int size)
+static void fsl_chan_set_request_count(struct fsl_dma_chan *fsl_chan, int size)
 {
-        if (size > 1024)
-                return;
+        BUG_ON(size > 1024);
+        DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
+                DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
+                        | ((__ilog2(size) << 24) & 0x0f000000),
+                32);
+}
 
-        if (size) {
-                DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
-                        DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
-                                | ((__ilog2(size) << 24) & 0x0f000000),
-                        32);
+/**
+ * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * @fsl_chan : Freescale DMA channel
+ * @enable   : 0 is disabled, 1 is enabled.
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA Request Count feature should be used in addition to this feature
+ * to set the number of bytes to transfer before pausing the channel.
+ */
+static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable)
+{
+        if (enable)
                 fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
-        } else
+        else
                 fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
 }
 
@@ -326,7 +339,8 @@ static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable)
 static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 {
         struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
-        struct fsl_desc_sw *desc;
+        struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
+        struct fsl_desc_sw *child;
         unsigned long flags;
         dma_cookie_t cookie;
 
@@ -334,7 +348,7 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
         spin_lock_irqsave(&fsl_chan->desc_lock, flags);
 
         cookie = fsl_chan->common.cookie;
-        list_for_each_entry(desc, &tx->tx_list, node) {
+        list_for_each_entry(child, &desc->tx_list, node) {
                 cookie++;
                 if (cookie < 0)
                         cookie = 1;
@@ -343,8 +357,8 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
         }
 
         fsl_chan->common.cookie = cookie;
-        append_ld_queue(fsl_chan, tx_to_fsl_desc(tx));
-        list_splice_init(&tx->tx_list, fsl_chan->ld_queue.prev);
+        append_ld_queue(fsl_chan, desc);
+        list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev);
 
         spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
 
@@ -366,6 +380,7 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
         desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc);
         if (desc_sw) {
                 memset(desc_sw, 0, sizeof(struct fsl_desc_sw));
+                INIT_LIST_HEAD(&desc_sw->tx_list);
                 dma_async_tx_descriptor_init(&desc_sw->async_tx,
                                                 &fsl_chan->common);
                 desc_sw->async_tx.tx_submit = fsl_dma_tx_submit;
@@ -455,7 +470,7 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags)
         new->async_tx.flags = flags;
 
         /* Insert the link descriptor to the LD ring */
-        list_add_tail(&new->node, &new->async_tx.tx_list);
+        list_add_tail(&new->node, &new->tx_list);
 
         /* Set End-of-link to the last link descriptor of new list*/
         set_ld_eol(fsl_chan, new);
@@ -513,7 +528,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
                 dma_dest += copy;
 
                 /* Insert the link descriptor to the LD ring */
-                list_add_tail(&new->node, &first->async_tx.tx_list);
+                list_add_tail(&new->node, &first->tx_list);
         } while (len);
 
         new->async_tx.flags = flags; /* client is in control of this ack */
@@ -528,7 +543,7 @@ fail:
         if (!first)
                 return NULL;
 
-        list = &first->async_tx.tx_list;
+        list = &first->tx_list;
         list_for_each_entry_safe_reverse(new, prev, list, node) {
                 list_del(&new->node);
                 dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
@@ -538,6 +553,229 @@ fail:
 }
 
 /**
+ * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: DMAEngine flags
+ *
+ * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
+ * DMA_SLAVE API, this gets the device-specific information from the
+ * chan->private variable.
+ */
+static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
+        struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
+        enum dma_data_direction direction, unsigned long flags)
+{
+        struct fsl_dma_chan *fsl_chan;
+        struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
+        struct fsl_dma_slave *slave;
+        struct list_head *tx_list;
+        size_t copy;
+
+        int i;
+        struct scatterlist *sg;
+        size_t sg_used;
+        size_t hw_used;
+        struct fsl_dma_hw_addr *hw;
+        dma_addr_t dma_dst, dma_src;
+
+        if (!chan)
+                return NULL;
+
+        if (!chan->private)
+                return NULL;
+
+        fsl_chan = to_fsl_chan(chan);
+        slave = chan->private;
+
+        if (list_empty(&slave->addresses))
+                return NULL;
+
+        hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
+        hw_used = 0;
+
+        /*
+         * Build the hardware transaction to copy from the scatterlist to
+         * the hardware, or from the hardware to the scatterlist
+         *
+         * If you are copying from the hardware to the scatterlist and it
+         * takes two hardware entries to fill an entire page, then both
+         * hardware entries will be coalesced into the same page
+         *
+         * If you are copying from the scatterlist to the hardware and a
+         * single page can fill two hardware entries, then the data will
+         * be read out of the page into the first hardware entry, and so on
+         */
+        for_each_sg(sgl, sg, sg_len, i) {
+                sg_used = 0;
+
+                /* Loop until the entire scatterlist entry is used */
+                while (sg_used < sg_dma_len(sg)) {
+
+                        /*
+                         * If we've used up the current hardware address/length
+                         * pair, we need to load a new one
+                         *
+                         * This is done in a while loop so that descriptors with
+                         * length == 0 will be skipped
+                         */
+                        while (hw_used >= hw->length) {
+
+                                /*
+                                 * If the current hardware entry is the last
+                                 * entry in the list, we're finished
+                                 */
+                                if (list_is_last(&hw->entry, &slave->addresses))
+                                        goto finished;
+
+                                /* Get the next hardware address/length pair */
+                                hw = list_entry(hw->entry.next,
+                                                struct fsl_dma_hw_addr, entry);
+                                hw_used = 0;
+                        }
+
+                        /* Allocate the link descriptor from DMA pool */
+                        new = fsl_dma_alloc_descriptor(fsl_chan);
+                        if (!new) {
+                                dev_err(fsl_chan->dev, "No free memory for "
+                                                       "link descriptor\n");
+                                goto fail;
+                        }
+#ifdef FSL_DMA_LD_DEBUG
+                        dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
+#endif
+
+                        /*
+                         * Calculate the maximum number of bytes to transfer,
+                         * making sure it is less than the DMA controller limit
+                         */
+                        copy = min_t(size_t, sg_dma_len(sg) - sg_used,
+                                             hw->length - hw_used);
+                        copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
+
+                        /*
+                         * DMA_FROM_DEVICE
+                         * from the hardware to the scatterlist
+                         *
+                         * DMA_TO_DEVICE
+                         * from the scatterlist to the hardware
+                         */
+                        if (direction == DMA_FROM_DEVICE) {
+                                dma_src = hw->address + hw_used;
+                                dma_dst = sg_dma_address(sg) + sg_used;
+                        } else {
+                                dma_src = sg_dma_address(sg) + sg_used;
+                                dma_dst = hw->address + hw_used;
+                        }
+
+                        /* Fill in the descriptor */
+                        set_desc_cnt(fsl_chan, &new->hw, copy);
+                        set_desc_src(fsl_chan, &new->hw, dma_src);
+                        set_desc_dest(fsl_chan, &new->hw, dma_dst);
+
+                        /*
+                         * If this is not the first descriptor, chain the
+                         * current descriptor after the previous descriptor
+                         */
+                        if (!first) {
+                                first = new;
+                        } else {
+                                set_desc_next(fsl_chan, &prev->hw,
+                                              new->async_tx.phys);
+                        }
+
+                        new->async_tx.cookie = 0;
+                        async_tx_ack(&new->async_tx);
+
+                        prev = new;
+                        sg_used += copy;
+                        hw_used += copy;
+
+                        /* Insert the link descriptor into the LD ring */
+                        list_add_tail(&new->node, &first->tx_list);
+                }
+        }
+
+finished:
+
+        /* All of the hardware address/length pairs had length == 0 */
+        if (!first || !new)
+                return NULL;
+
+        new->async_tx.flags = flags;
+        new->async_tx.cookie = -EBUSY;
+
+        /* Set End-of-link to the last link descriptor of new list */
+        set_ld_eol(fsl_chan, new);
+
+        /* Enable extra controller features */
+        if (fsl_chan->set_src_loop_size)
+                fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size);
+
+        if (fsl_chan->set_dest_loop_size)
+                fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size);
+
+        if (fsl_chan->toggle_ext_start)
+                fsl_chan->toggle_ext_start(fsl_chan, slave->external_start);
+
+        if (fsl_chan->toggle_ext_pause)
+                fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause);
+
+        if (fsl_chan->set_request_count)
+                fsl_chan->set_request_count(fsl_chan, slave->request_count);
+
+        return &first->async_tx;
+
+fail:
+        /* If first was not set, then we failed to allocate the very first
+         * descriptor, and we're done */
+        if (!first)
+                return NULL;
+
+        /*
+         * First is set, so all of the descriptors we allocated have been added
+         * to first->tx_list, INCLUDING "first" itself. Therefore we
+         * must traverse the list backwards freeing each descriptor in turn
+         *
+         * We're re-using variables for the loop, oh well
+         */
+        tx_list = &first->tx_list;
+        list_for_each_entry_safe_reverse(new, prev, tx_list, node) {
+                list_del_init(&new->node);
+                dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
+        }
+
+        return NULL;
+}
+
+static void fsl_dma_device_terminate_all(struct dma_chan *chan)
+{
+        struct fsl_dma_chan *fsl_chan;
+        struct fsl_desc_sw *desc, *tmp;
+        unsigned long flags;
+
+        if (!chan)
+                return;
+
+        fsl_chan = to_fsl_chan(chan);
+
+        /* Halt the DMA engine */
+        dma_halt(fsl_chan);
+
+        spin_lock_irqsave(&fsl_chan->desc_lock, flags);
+
+        /* Remove and free all of the descriptors in the LD queue */
+        list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) {
+                list_del(&desc->node);
+                dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
+        }
+
+        spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
+}
+
+/**
  * fsl_dma_update_completed_cookie - Update the completed cookie.
  * @fsl_chan : Freescale DMA channel
  */
@@ -883,6 +1121,7 @@ static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev,
                 new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
                 new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size;
                 new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size;
+                new_fsl_chan->set_request_count = fsl_chan_set_request_count;
         }
 
         spin_lock_init(&new_fsl_chan->desc_lock);
@@ -962,12 +1201,15 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
 
         dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
         dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
+        dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
         fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
         fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
         fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
         fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
         fdev->common.device_is_tx_complete = fsl_dma_is_complete;
         fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
+        fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
+        fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
         fdev->common.dev = &dev->dev;
 
         fdev->irq = irq_of_parse_and_map(dev->node, 0);
diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h
index dc7f26865797..0df14cbb8ca3 100644
--- a/drivers/dma/fsldma.h
+++ b/drivers/dma/fsldma.h
@@ -90,6 +90,7 @@ struct fsl_dma_ld_hw {
 struct fsl_desc_sw {
         struct fsl_dma_ld_hw hw;
         struct list_head node;
+        struct list_head tx_list;
         struct dma_async_tx_descriptor async_tx;
         struct list_head *ld;
         void *priv;
@@ -143,10 +144,11 @@ struct fsl_dma_chan {
         struct tasklet_struct tasklet;
         u32 feature;
 
-        void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int size);
+        void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int enable);
         void (*toggle_ext_start)(struct fsl_dma_chan *fsl_chan, int enable);
         void (*set_src_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
         void (*set_dest_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
+        void (*set_request_count)(struct fsl_dma_chan *fsl_chan, int size);
 };
 
 #define to_fsl_chan(chan) container_of(chan, struct fsl_dma_chan, common)
diff --git a/drivers/dma/ioat.c b/drivers/dma/ioat.c
deleted file mode 100644
index 2225bb6ba3d1..000000000000
--- a/drivers/dma/ioat.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Intel I/OAT DMA Linux driver
- * Copyright(c) 2007 - 2009 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- */
-
-/*
- * This driver supports an Intel I/OAT DMA engine, which does asynchronous
- * copy operations.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/dca.h>
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
-#include "ioatdma_hw.h"
-
-MODULE_VERSION(IOAT_DMA_VERSION);
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Intel Corporation");
-
-static struct pci_device_id ioat_pci_tbl[] = {
-        /* I/OAT v1 platforms */
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB)  },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
-        { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
-
-        /* I/OAT v2 platforms */
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
-
-        /* I/OAT v3 platforms */
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
-        { 0, }
-};
-
-struct ioat_device {
-        struct pci_dev          *pdev;
-        void __iomem            *iobase;
-        struct ioatdma_device   *dma;
-        struct dca_provider     *dca;
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
-                                const struct pci_device_id *id);
-static void __devexit ioat_remove(struct pci_dev *pdev);
-
-static int ioat_dca_enabled = 1;
-module_param(ioat_dca_enabled, int, 0644);
-MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
-
-static struct pci_driver ioat_pci_driver = {
-        .name           = "ioatdma",
-        .id_table       = ioat_pci_tbl,
-        .probe          = ioat_probe,
-        .remove         = __devexit_p(ioat_remove),
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
-                                const struct pci_device_id *id)
-{
-        void __iomem *iobase;
-        struct ioat_device *device;
-        unsigned long mmio_start, mmio_len;
-        int err;
-
-        err = pci_enable_device(pdev);
-        if (err)
-                goto err_enable_device;
-
-        err = pci_request_regions(pdev, ioat_pci_driver.name);
-        if (err)
-                goto err_request_regions;
-
-        err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
-        if (err)
-                err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
-        if (err)
-                goto err_set_dma_mask;
-
-        err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
-        if (err)
-                err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
-        if (err)
-                goto err_set_dma_mask;
-
-        mmio_start = pci_resource_start(pdev, 0);
-        mmio_len = pci_resource_len(pdev, 0);
-        iobase = ioremap(mmio_start, mmio_len);
-        if (!iobase) {
-                err = -ENOMEM;
-                goto err_ioremap;
-        }
-
-        device = kzalloc(sizeof(*device), GFP_KERNEL);
-        if (!device) {
-                err = -ENOMEM;
-                goto err_kzalloc;
-        }
-        device->pdev = pdev;
-        pci_set_drvdata(pdev, device);
-        device->iobase = iobase;
-
-        pci_set_master(pdev);
-
-        switch (readb(iobase + IOAT_VER_OFFSET)) {
-        case IOAT_VER_1_2:
-                device->dma = ioat_dma_probe(pdev, iobase);
-                if (device->dma && ioat_dca_enabled)
-                        device->dca = ioat_dca_init(pdev, iobase);
-                break;
-        case IOAT_VER_2_0:
-                device->dma = ioat_dma_probe(pdev, iobase);
-                if (device->dma && ioat_dca_enabled)
-                        device->dca = ioat2_dca_init(pdev, iobase);
-                break;
-        case IOAT_VER_3_0:
-                device->dma = ioat_dma_probe(pdev, iobase);
-                if (device->dma && ioat_dca_enabled)
-                        device->dca = ioat3_dca_init(pdev, iobase);
-                break;
-        default:
-                err = -ENODEV;
-                break;
-        }
-        if (!device->dma)
-                err = -ENODEV;
-
-        if (err)
-                goto err_version;
-
-        return 0;
-
-err_version:
-        kfree(device);
-err_kzalloc:
-        iounmap(iobase);
-err_ioremap:
-err_set_dma_mask:
-        pci_release_regions(pdev);
-        pci_disable_device(pdev);
-err_request_regions:
-err_enable_device:
-        return err;
-}
-
-static void __devexit ioat_remove(struct pci_dev *pdev)
-{
-        struct ioat_device *device = pci_get_drvdata(pdev);
-
-        dev_err(&pdev->dev, "Removing dma and dca services\n");
-        if (device->dca) {
-                unregister_dca_provider(device->dca);
-                free_dca_provider(device->dca);
-                device->dca = NULL;
-        }
-
-        if (device->dma) {
-                ioat_dma_remove(device->dma);
-                device->dma = NULL;
-        }
-
-        kfree(device);
-}
-
-static int __init ioat_init_module(void)
-{
-        return pci_register_driver(&ioat_pci_driver);
-}
-module_init(ioat_init_module);
-
-static void __exit ioat_exit_module(void)
-{
-        pci_unregister_driver(&ioat_pci_driver);
-}
-module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
new file mode 100644
index 000000000000..8997d3fb9051
--- /dev/null
+++ b/drivers/dma/ioat/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
+ioatdma-objs := pci.o dma.o dma_v2.o dma_v3.o dca.o
diff --git a/drivers/dma/ioat_dca.c b/drivers/dma/ioat/dca.c
index c012a1e15043..69d02615c4d6 100644
--- a/drivers/dma/ioat_dca.c
+++ b/drivers/dma/ioat/dca.c
@@ -33,8 +33,8 @@
 #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
 #endif
 
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
+#include "dma.h"
+#include "registers.h"
 
 /*
  * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
@@ -242,7 +242,8 @@ static struct dca_ops ioat_dca_ops = {
 };
 
 
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
         struct dca_provider *dca;
         struct ioat_dca_priv *ioatdca;
@@ -407,7 +408,8 @@ static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
         return slots;
 }
 
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
         struct dca_provider *dca;
         struct ioat_dca_priv *ioatdca;
@@ -602,7 +604,8 @@ static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
         return slots;
 }
 
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
         struct dca_provider *dca;
         struct ioat_dca_priv *ioatdca;
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
new file mode 100644
index 000000000000..c524d36d3c2e
--- /dev/null
+++ b/drivers/dma/ioat/dma.c
@@ -0,0 +1,1238 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2004 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine, which does asynchronous
+ * copy operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/workqueue.h>
+#include <linux/i7300_idle.h>
+#include "dma.h"
+#include "registers.h"
+#include "hw.h"
+
+int ioat_pending_level = 4;
+module_param(ioat_pending_level, int, 0644);
+MODULE_PARM_DESC(ioat_pending_level,
+                 "high-water mark for pushing ioat descriptors (default: 4)");
+
+/* internal functions */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat);
+static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
+
+/**
+ * ioat_dma_do_interrupt - handler used for single vector interrupt mode
+ * @irq: interrupt id
+ * @data: interrupt data
+ */
+static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
+{
+        struct ioatdma_device *instance = data;
+        struct ioat_chan_common *chan;
+        unsigned long attnstatus;
+        int bit;
+        u8 intrctrl;
+
+        intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
+
+        if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
+                return IRQ_NONE;
+
+        if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
+                writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
+                return IRQ_NONE;
+        }
+
+        attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
+        for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
+                chan = ioat_chan_by_index(instance, bit);
+                tasklet_schedule(&chan->cleanup_task);
+        }
+
+        writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
+        return IRQ_HANDLED;
+}
+
+/**
+ * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
+ * @irq: interrupt id
+ * @data: interrupt data
+ */
+static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
+{
+        struct ioat_chan_common *chan = data;
+
+        tasklet_schedule(&chan->cleanup_task);
+
+        return IRQ_HANDLED;
+}
+
+static void ioat1_cleanup_tasklet(unsigned long data);
+
+/* common channel initialization */
+void ioat_init_channel(struct ioatdma_device *device,
+                       struct ioat_chan_common *chan, int idx,
+                       void (*timer_fn)(unsigned long),
+                       void (*tasklet)(unsigned long),
+                       unsigned long ioat)
+{
+        struct dma_device *dma = &device->common;
+
+        chan->device = device;
+        chan->reg_base = device->reg_base + (0x80 * (idx + 1));
+        spin_lock_init(&chan->cleanup_lock);
+        chan->common.device = dma;
+        list_add_tail(&chan->common.device_node, &dma->channels);
+        device->idx[idx] = chan;
+        init_timer(&chan->timer);
+        chan->timer.function = timer_fn;
+        chan->timer.data = ioat;
+        tasklet_init(&chan->cleanup_task, tasklet, ioat);
+        tasklet_disable(&chan->cleanup_task);
+}
+
+static void ioat1_timer_event(unsigned long data);
+
+/**
+ * ioat1_dma_enumerate_channels - find and initialize the device's channels
+ * @device: the device to be enumerated
+ */
+static int ioat1_enumerate_channels(struct ioatdma_device *device)
+{
+        u8 xfercap_scale;
+        u32 xfercap;
+        int i;
+        struct ioat_dma_chan *ioat;
+        struct device *dev = &device->pdev->dev;
+        struct dma_device *dma = &device->common;
+
+        INIT_LIST_HEAD(&dma->channels);
+        dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+        dma->chancnt &= 0x1f; /* bits [4:0] valid */
+        if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+                dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+                         dma->chancnt, ARRAY_SIZE(device->idx));
+                dma->chancnt = ARRAY_SIZE(device->idx);
+        }
+        xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+        xfercap_scale &= 0x1f; /* bits [4:0] valid */
+        xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
+        dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
+
+#ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
+        if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+                dma->chancnt--;
+#endif
+        for (i = 0; i < dma->chancnt; i++) {
+                ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
+                if (!ioat)
+                        break;
+
+                ioat_init_channel(device, &ioat->base, i,
+                                  ioat1_timer_event,
+                                  ioat1_cleanup_tasklet,
+                                  (unsigned long) ioat);
+                ioat->xfercap = xfercap;
+                spin_lock_init(&ioat->desc_lock);
+                INIT_LIST_HEAD(&ioat->free_desc);
+                INIT_LIST_HEAD(&ioat->used_desc);
+        }
+        dma->chancnt = i;
+        return i;
+}
+
+/**
+ * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
+ *                                 descriptors to hw
+ * @chan: DMA channel handle
+ */
+static inline void
+__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
+{
+        void __iomem *reg_base = ioat->base.reg_base;
+
+        dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
+                __func__, ioat->pending);
+        ioat->pending = 0;
+        writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
+}
+
+static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(chan);
+
+        if (ioat->pending > 0) {
+                spin_lock_bh(&ioat->desc_lock);
+                __ioat1_dma_memcpy_issue_pending(ioat);
+                spin_unlock_bh(&ioat->desc_lock);
+        }
+}
+
+/**
+ * ioat1_reset_channel - restart a channel
+ * @ioat: IOAT DMA channel handle
+ */
+static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        void __iomem *reg_base = chan->reg_base;
+        u32 chansts, chanerr;
+
+        dev_warn(to_dev(chan), "reset\n");
+        chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
+        chansts = *chan->completion & IOAT_CHANSTS_STATUS;
+        if (chanerr) {
+                dev_err(to_dev(chan),
+                        "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
+                        chan_num(chan), chansts, chanerr);
+                writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
+        }
+
+        /*
+         * whack it upside the head with a reset
+         * and wait for things to settle out.
+         * force the pending count to a really big negative
+         * to make sure no one forces an issue_pending
+         * while we're waiting.
+         */
+
+        ioat->pending = INT_MIN;
+        writeb(IOAT_CHANCMD_RESET,
+               reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+        set_bit(IOAT_RESET_PENDING, &chan->state);
+        mod_timer(&chan->timer, jiffies + RESET_DELAY);
+}
+
+static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+        struct dma_chan *c = tx->chan;
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_desc_sw *first;
+        struct ioat_desc_sw *chain_tail;
+        dma_cookie_t cookie;
+
+        spin_lock_bh(&ioat->desc_lock);
+        /* cookie incr and addition to used_list must be atomic */
+        cookie = c->cookie;
+        cookie++;
+        if (cookie < 0)
+                cookie = 1;
+        c->cookie = cookie;
+        tx->cookie = cookie;
+        dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+
+        /* write address into NextDescriptor field of last desc in chain */
+        first = to_ioat_desc(desc->tx_list.next);
+        chain_tail = to_ioat_desc(ioat->used_desc.prev);
+        /* make descriptor updates globally visible before chaining */
+        wmb();
+        chain_tail->hw->next = first->txd.phys;
+        list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
+        dump_desc_dbg(ioat, chain_tail);
+        dump_desc_dbg(ioat, first);
+
+        if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+                mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+        ioat->active += desc->hw->tx_cnt;
+        ioat->pending += desc->hw->tx_cnt;
+        if (ioat->pending >= ioat_pending_level)
+                __ioat1_dma_memcpy_issue_pending(ioat);
+        spin_unlock_bh(&ioat->desc_lock);
+
+        return cookie;
+}
+
+/**
+ * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
+ * @ioat: the channel supplying the memory pool for the descriptors
+ * @flags: allocation flags
+ */
+static struct ioat_desc_sw *
+ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
+{
+        struct ioat_dma_descriptor *desc;
+        struct ioat_desc_sw *desc_sw;
+        struct ioatdma_device *ioatdma_device;
+        dma_addr_t phys;
+
+        ioatdma_device = ioat->base.device;
+        desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
+        if (unlikely(!desc))
+                return NULL;
+
+        desc_sw = kzalloc(sizeof(*desc_sw), flags);
+        if (unlikely(!desc_sw)) {
+                pci_pool_free(ioatdma_device->dma_pool, desc, phys);
+                return NULL;
+        }
+
+        memset(desc, 0, sizeof(*desc));
+
+        INIT_LIST_HEAD(&desc_sw->tx_list);
+        dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
+        desc_sw->txd.tx_submit = ioat1_tx_submit;
+        desc_sw->hw = desc;
+        desc_sw->txd.phys = phys;
+        set_desc_id(desc_sw, -1);
+
+        return desc_sw;
+}
+
+static int ioat_initial_desc_count = 256;
+module_param(ioat_initial_desc_count, int, 0644);
+MODULE_PARM_DESC(ioat_initial_desc_count,
+                 "ioat1: initial descriptors per channel (default: 256)");
+/**
+ * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
+ * @chan: the channel to be filled out
+ */
+static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_desc_sw *desc;
+        u32 chanerr;
+        int i;
+        LIST_HEAD(tmp_list);
+
+        /* have we already been set up? */
+        if (!list_empty(&ioat->free_desc))
+                return ioat->desccount;
+
+        /* Setup register to interrupt and write completion status on error */
+        writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+
+        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+        if (chanerr) {
+                dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
+                writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+        }
+
+        /* Allocate descriptors */
+        for (i = 0; i < ioat_initial_desc_count; i++) {
+                desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
+                if (!desc) {
+                        dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
+                        break;
+                }
+                set_desc_id(desc, i);
+                list_add_tail(&desc->node, &tmp_list);
+        }
+        spin_lock_bh(&ioat->desc_lock);
+        ioat->desccount = i;
+        list_splice(&tmp_list, &ioat->free_desc);
+        spin_unlock_bh(&ioat->desc_lock);
+
+        /* allocate a completion writeback area */
+        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
+        chan->completion = pci_pool_alloc(chan->device->completion_pool,
+                                          GFP_KERNEL, &chan->completion_dma);
+        memset(chan->completion, 0, sizeof(*chan->completion));
+        writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+        writel(((u64) chan->completion_dma) >> 32,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
+        tasklet_enable(&chan->cleanup_task);
+        ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */
+        dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+                __func__, ioat->desccount);
+        return ioat->desccount;
+}
+
+/**
+ * ioat1_dma_free_chan_resources - release all the descriptors
+ * @chan: the channel to be cleaned
+ */
+static void ioat1_dma_free_chan_resources(struct dma_chan *c)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioatdma_device *ioatdma_device = chan->device;
+        struct ioat_desc_sw *desc, *_desc;
+        int in_use_descs = 0;
+
+        /* Before freeing channel resources first check
+         * if they have been previously allocated for this channel.
+         */
+        if (ioat->desccount == 0)
+                return;
+
+        tasklet_disable(&chan->cleanup_task);
+        del_timer_sync(&chan->timer);
+        ioat1_cleanup(ioat);
+
+        /* Delay 100ms after reset to allow internal DMA logic to quiesce
+         * before removing DMA descriptor resources.
+         */
+        writeb(IOAT_CHANCMD_RESET,
+               chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+        mdelay(100);
+
+        spin_lock_bh(&ioat->desc_lock);
+        list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
+                dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
+                        __func__, desc_id(desc));
+                dump_desc_dbg(ioat, desc);
+                in_use_descs++;
+                list_del(&desc->node);
+                pci_pool_free(ioatdma_device->dma_pool, desc->hw,
+                              desc->txd.phys);
+                kfree(desc);
+        }
+        list_for_each_entry_safe(desc, _desc,
+                                 &ioat->free_desc, node) {
+                list_del(&desc->node);
+                pci_pool_free(ioatdma_device->dma_pool, desc->hw,
+                              desc->txd.phys);
+                kfree(desc);
+        }
+        spin_unlock_bh(&ioat->desc_lock);
+
+        pci_pool_free(ioatdma_device->completion_pool,
+                      chan->completion,
+                      chan->completion_dma);
+
+        /* one is ok since we left it on there on purpose */
+        if (in_use_descs > 1)
+                dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
+                        in_use_descs - 1);
+
+        chan->last_completion = 0;
+        chan->completion_dma = 0;
+        ioat->pending = 0;
+        ioat->desccount = 0;
+}
+
+/**
+ * ioat1_dma_get_next_descriptor - return the next available descriptor
+ * @ioat: IOAT DMA channel handle
+ *
+ * Gets the next descriptor from the chain, and must be called with the
+ * channel's desc_lock held.  Allocates more descriptors if the channel
+ * has run out.
+ */
+static struct ioat_desc_sw *
+ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
+{
+        struct ioat_desc_sw *new;
+
+        if (!list_empty(&ioat->free_desc)) {
+                new = to_ioat_desc(ioat->free_desc.next);
+                list_del(&new->node);
+        } else {
+                /* try to get another desc */
+                new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
+                if (!new) {
+                        dev_err(to_dev(&ioat->base), "alloc failed\n");
+                        return NULL;
+                }
+        }
+        dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
+                __func__, desc_id(new));
+        prefetch(new->hw);
+        return new;
+}
+
+static struct dma_async_tx_descriptor *
+ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
+                      dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_desc_sw *desc;
+        size_t copy;
+        LIST_HEAD(chain);
+        dma_addr_t src = dma_src;
+        dma_addr_t dest = dma_dest;
+        size_t total_len = len;
+        struct ioat_dma_descriptor *hw = NULL;
+        int tx_cnt = 0;
+
+        spin_lock_bh(&ioat->desc_lock);
+        desc = ioat1_dma_get_next_descriptor(ioat);
+        do {
+                if (!desc)
+                        break;
+
+                tx_cnt++;
+                copy = min_t(size_t, len, ioat->xfercap);
+
+                hw = desc->hw;
+                hw->size = copy;
+                hw->ctl = 0;
+                hw->src_addr = src;
+                hw->dst_addr = dest;
+
+                list_add_tail(&desc->node, &chain);
+
+                len -= copy;
+                dest += copy;
+                src += copy;
+                if (len) {
+                        struct ioat_desc_sw *next;
+
+                        async_tx_ack(&desc->txd);
+                        next = ioat1_dma_get_next_descriptor(ioat);
+                        hw->next = next ? next->txd.phys : 0;
+                        dump_desc_dbg(ioat, desc);
+                        desc = next;
+                } else
+                        hw->next = 0;
+        } while (len);
+
+        if (!desc) {
+                struct ioat_chan_common *chan = &ioat->base;
+
+                dev_err(to_dev(chan),
+                        "chan%d - get_next_desc failed\n", chan_num(chan));
+                list_splice(&chain, &ioat->free_desc);
+                spin_unlock_bh(&ioat->desc_lock);
+                return NULL;
+        }
+        spin_unlock_bh(&ioat->desc_lock);
+
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        list_splice(&chain, &desc->tx_list);
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.compl_write = 1;
+        hw->tx_cnt = tx_cnt;
+        dump_desc_dbg(ioat, desc);
+
+        return &desc->txd;
+}
+
+static void ioat1_cleanup_tasklet(unsigned long data)
+{
+        struct ioat_dma_chan *chan = (void *)data;
+
+        ioat1_cleanup(chan);
+        writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
+                    size_t len, struct ioat_dma_descriptor *hw)
+{
+        struct pci_dev *pdev = chan->device->pdev;
+        size_t offset = len - hw->size;
+
+        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                ioat_unmap(pdev, hw->dst_addr - offset, len,
+                           PCI_DMA_FROMDEVICE, flags, 1);
+
+        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
+                ioat_unmap(pdev, hw->src_addr - offset, len,
+                           PCI_DMA_TODEVICE, flags, 0);
+}
+
+unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
+{
+        unsigned long phys_complete;
+        u64 completion;
+
+        completion = *chan->completion;
+        phys_complete = ioat_chansts_to_addr(completion);
+
+        dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
+                (unsigned long long) phys_complete);
+
+        if (is_ioat_halted(completion)) {
+                u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
+                        chanerr);
+
+                /* TODO do something to salvage the situation */
+        }
+
+        return phys_complete;
+}
+
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+                           unsigned long *phys_complete)
+{
+        *phys_complete = ioat_get_current_completion(chan);
+        if (*phys_complete == chan->last_completion)
+                return false;
+        clear_bit(IOAT_COMPLETION_ACK, &chan->state);
+        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+        return true;
+}
+
+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct list_head *_desc, *n;
+        struct dma_async_tx_descriptor *tx;
+
+        dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
+                 __func__, phys_complete);
+        list_for_each_safe(_desc, n, &ioat->used_desc) {
+                struct ioat_desc_sw *desc;
+
+                prefetch(n);
+                desc = list_entry(_desc, typeof(*desc), node);
+                tx = &desc->txd;
+                /*
+                 * Incoming DMA requests may use multiple descriptors,
+                 * due to exceeding xfercap, perhaps. If so, only the
+                 * last one will have a cookie, and require unmapping.
+                 */
+                dump_desc_dbg(ioat, desc);
+                if (tx->cookie) {
+                        chan->completed_cookie = tx->cookie;
+                        tx->cookie = 0;
+                        ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+                        ioat->active -= desc->hw->tx_cnt;
+                        if (tx->callback) {
+                                tx->callback(tx->callback_param);
+                                tx->callback = NULL;
+                        }
+                }
+
+                if (tx->phys != phys_complete) {
+                        /*
+                         * a completed entry, but not the last, so clean
+                         * up if the client is done with the descriptor
+                         */
+                        if (async_tx_test_ack(tx))
+                                list_move_tail(&desc->node, &ioat->free_desc);
+                } else {
+                        /*
+                         * last used desc. Do not remove, so we can
+                         * append from it.
+                         */
+
+                        /* if nothing else is pending, cancel the
+                         * completion timeout
+                         */
+                        if (n == &ioat->used_desc) {
+                                dev_dbg(to_dev(chan),
+                                        "%s cancel completion timeout\n",
+                                        __func__);
+                                clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                        }
+
+                        /* TODO check status bits? */
+                        break;
+                }
+        }
+
+        chan->last_completion = phys_complete;
+}
+
+/**
+ * ioat1_cleanup - cleanup up finished descriptors
+ * @chan: ioat channel to be cleaned up
+ *
+ * To prevent lock contention we defer cleanup when the locks are
+ * contended with a terminal timeout that forces cleanup and catches
+ * completion notification errors.
+ */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        if (!spin_trylock_bh(&chan->cleanup_lock))
+                return;
+
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        if (!spin_trylock_bh(&ioat->desc_lock)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->desc_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat1_timer_event(unsigned long data)
+{
+        struct ioat_dma_chan *ioat = (void *) data;
+        struct ioat_chan_common *chan = &ioat->base;
+
+        dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
+                struct ioat_desc_sw *desc;
+
+                spin_lock_bh(&ioat->desc_lock);
+
+                /* restart active descriptors */
+                desc = to_ioat_desc(ioat->used_desc.prev);
+                ioat_set_chainaddr(ioat, desc->txd.phys);
+                ioat_start(chan);
+
+                ioat->pending = 0;
+                set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                spin_unlock_bh(&ioat->desc_lock);
+        } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+                unsigned long phys_complete;
+
+                spin_lock_bh(&ioat->desc_lock);
+                /* if we haven't made progress and we have already
+                 * acknowledged a pending completion once, then be more
+                 * forceful with a restart
+                 */
+                if (ioat_cleanup_preamble(chan, &phys_complete))
+                        __cleanup(ioat, phys_complete);
+                else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                        ioat1_reset_channel(ioat);
+                else {
+                        u64 status = ioat_chansts(chan);
+
+                        /* manually update the last completion address */
+                        if (ioat_chansts_to_addr(status) != 0)
+                                *chan->completion = status;
+
+                        set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                }
+                spin_unlock_bh(&ioat->desc_lock);
+        }
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static enum dma_status
+ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                      dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+                return DMA_SUCCESS;
+
+        ioat1_cleanup(ioat);
+
+        return ioat_is_complete(c, cookie, done, used);
+}
+
+static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_desc_sw *desc;
+        struct ioat_dma_descriptor *hw;
+
+        spin_lock_bh(&ioat->desc_lock);
+
+        desc = ioat1_dma_get_next_descriptor(ioat);
+
+        if (!desc) {
+                dev_err(to_dev(chan),
+                        "Unable to start null desc - get next desc failed\n");
+                spin_unlock_bh(&ioat->desc_lock);
+                return;
+        }
+
+        hw = desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = 1;
+        hw->ctl_f.compl_write = 1;
+        /* set size to non-zero value (channel returns error when size is 0) */
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        hw->src_addr = 0;
+        hw->dst_addr = 0;
+        async_tx_ack(&desc->txd);
+        hw->next = 0;
+        list_add_tail(&desc->node, &ioat->used_desc);
+        dump_desc_dbg(ioat, desc);
+
+        ioat_set_chainaddr(ioat, desc->txd.phys);
+        ioat_start(chan);
+        spin_unlock_bh(&ioat->desc_lock);
+}
+
+/*
+ * Perform a IOAT transaction to verify the HW works.
+ */
+#define IOAT_TEST_SIZE 2000
+
+static void __devinit ioat_dma_test_callback(void *dma_async_param)
+{
+        struct completion *cmp = dma_async_param;
+
+        complete(cmp);
+}
+
+/**
+ * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
+ * @device: device to be tested
+ */
+int __devinit ioat_dma_self_test(struct ioatdma_device *device)
+{
+        int i;
+        u8 *src;
+        u8 *dest;
+        struct dma_device *dma = &device->common;
+        struct device *dev = &device->pdev->dev;
+        struct dma_chan *dma_chan;
+        struct dma_async_tx_descriptor *tx;
+        dma_addr_t dma_dest, dma_src;
+        dma_cookie_t cookie;
+        int err = 0;
+        struct completion cmp;
+        unsigned long tmo;
+        unsigned long flags;
+
+        src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
+        if (!src)
+                return -ENOMEM;
+        dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
+        if (!dest) {
+                kfree(src);
+                return -ENOMEM;
+        }
+
+        /* Fill in src buffer */
+        for (i = 0; i < IOAT_TEST_SIZE; i++)
+                src[i] = (u8)i;
+
+        /* Start copy, using first DMA channel */
+        dma_chan = container_of(dma->channels.next, struct dma_chan,
+                                device_node);
+        if (dma->device_alloc_chan_resources(dma_chan) < 1) {
+                dev_err(dev, "selftest cannot allocate chan resource\n");
+                err = -ENODEV;
+                goto out;
+        }
+
+        dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
+        dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
+        flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
+                DMA_PREP_INTERRUPT;
+        tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
+                                                   IOAT_TEST_SIZE, flags);
+        if (!tx) {
+                dev_err(dev, "Self-test prep failed, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test setup failed, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (tmo == 0 ||
+            dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
+                                        != DMA_SUCCESS) {
+                dev_err(dev, "Self-test copy timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        if (memcmp(src, dest, IOAT_TEST_SIZE)) {
+                dev_err(dev, "Self-test copy failed compare, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+free_resources:
+        dma->device_free_chan_resources(dma_chan);
+out:
+        kfree(src);
+        kfree(dest);
+        return err;
+}
+
+static char ioat_interrupt_style[32] = "msix";
+module_param_string(ioat_interrupt_style, ioat_interrupt_style,
+                    sizeof(ioat_interrupt_style), 0644);
+MODULE_PARM_DESC(ioat_interrupt_style,
+                 "set ioat interrupt style: msix (default), "
+                 "msix-single-vector, msi, intx)");
+
+/**
+ * ioat_dma_setup_interrupts - setup interrupt handler
+ * @device: ioat device
+ */
+static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
+{
+        struct ioat_chan_common *chan;
+        struct pci_dev *pdev = device->pdev;
+        struct device *dev = &pdev->dev;
+        struct msix_entry *msix;
+        int i, j, msixcnt;
+        int err = -EINVAL;
+        u8 intrctrl = 0;
+
+        if (!strcmp(ioat_interrupt_style, "msix"))
+                goto msix;
+        if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
+                goto msix_single_vector;
+        if (!strcmp(ioat_interrupt_style, "msi"))
+                goto msi;
+        if (!strcmp(ioat_interrupt_style, "intx"))
+                goto intx;
+        dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
+        goto err_no_irq;
+
+msix:
+        /* The number of MSI-X vectors should equal the number of channels */
+        msixcnt = device->common.chancnt;
+        for (i = 0; i < msixcnt; i++)
+                device->msix_entries[i].entry = i;
+
+        err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
+        if (err < 0)
+                goto msi;
+        if (err > 0)
+                goto msix_single_vector;
+
+        for (i = 0; i < msixcnt; i++) {
+                msix = &device->msix_entries[i];
+                chan = ioat_chan_by_index(device, i);
+                err = devm_request_irq(dev, msix->vector,
+                                       ioat_dma_do_interrupt_msix, 0,
+                                       "ioat-msix", chan);
+                if (err) {
+                        for (j = 0; j < i; j++) {
+                                msix = &device->msix_entries[j];
+                                chan = ioat_chan_by_index(device, j);
+                                devm_free_irq(dev, msix->vector, chan);
+                        }
+                        goto msix_single_vector;
+                }
+        }
+        intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
+        goto done;
+
+msix_single_vector:
+        msix = &device->msix_entries[0];
+        msix->entry = 0;
+        err = pci_enable_msix(pdev, device->msix_entries, 1);
+        if (err)
+                goto msi;
+
+        err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
+                               "ioat-msix", device);
+        if (err) {
+                pci_disable_msix(pdev);
+                goto msi;
+        }
+        goto done;
+
+msi:
+        err = pci_enable_msi(pdev);
+        if (err)
+                goto intx;
+
+        err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
+                               "ioat-msi", device);
+        if (err) {
+                pci_disable_msi(pdev);
+                goto intx;
+        }
+        goto done;
+
+intx:
+        err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
+                               IRQF_SHARED, "ioat-intx", device);
+        if (err)
+                goto err_no_irq;
+
+done:
+        if (device->intr_quirk)
+                device->intr_quirk(device);
+        intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
+        writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
+        return 0;
+
+err_no_irq:
+        /* Disable all interrupt generation */
+        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
+        dev_err(dev, "no usable interrupts\n");
+        return err;
+}
+
+static void ioat_disable_interrupts(struct ioatdma_device *device)
+{
+        /* Disable all interrupt generation */
+        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
+}
+
+int __devinit ioat_probe(struct ioatdma_device *device)
+{
+        int err = -ENODEV;
+        struct dma_device *dma = &device->common;
+        struct pci_dev *pdev = device->pdev;
+        struct device *dev = &pdev->dev;
+
+        /* DMA coherent memory pool for DMA descriptor allocations */
+        device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
+                                           sizeof(struct ioat_dma_descriptor),
+                                           64, 0);
+        if (!device->dma_pool) {
+                err = -ENOMEM;
+                goto err_dma_pool;
+        }
+
+        device->completion_pool = pci_pool_create("completion_pool", pdev,
+                                                  sizeof(u64), SMP_CACHE_BYTES,
+                                                  SMP_CACHE_BYTES);
+
+        if (!device->completion_pool) {
+                err = -ENOMEM;
+                goto err_completion_pool;
+        }
+
+        device->enumerate_channels(device);
+
+        dma_cap_set(DMA_MEMCPY, dma->cap_mask);
+        dma->dev = &pdev->dev;
+
+        if (!dma->chancnt) {
+                dev_err(dev, "zero channels detected\n");
+                goto err_setup_interrupts;
+        }
+
+        err = ioat_dma_setup_interrupts(device);
+        if (err)
+                goto err_setup_interrupts;
+
+        err = device->self_test(device);
+        if (err)
+                goto err_self_test;
+
+        return 0;
+
+err_self_test:
+        ioat_disable_interrupts(device);
+err_setup_interrupts:
+        pci_pool_destroy(device->completion_pool);
+err_completion_pool:
+        pci_pool_destroy(device->dma_pool);
+err_dma_pool:
+        return err;
+}
+
+int __devinit ioat_register(struct ioatdma_device *device)
+{
+        int err = dma_async_device_register(&device->common);
+
+        if (err) {
+                ioat_disable_interrupts(device);
+                pci_pool_destroy(device->completion_pool);
+                pci_pool_destroy(device->dma_pool);
+        }
+
+        return err;
+}
+
+/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
+static void ioat1_intr_quirk(struct ioatdma_device *device)
+{
+        struct pci_dev *pdev = device->pdev;
+        u32 dmactrl;
+
+        pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
+        if (pdev->msi_enabled)
+                dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
+        else
+                dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
+        pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
+}
+
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+        return sprintf(page, "%d\n", ioat->desccount);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+        return sprintf(page, "%d\n", ioat->active);
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static ssize_t cap_show(struct dma_chan *c, char *page)
+{
+        struct dma_device *dma = c->device;
+
+        return sprintf(page, "copy%s%s%s%s%s%s\n",
+                       dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
+                       dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
+                       dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
+                       dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
+                       dma_has_cap(DMA_MEMSET, dma->cap_mask)  ? " fill" : "",
+                       dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
+
+}
+struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
+
+static ssize_t version_show(struct dma_chan *c, char *page)
+{
+        struct dma_device *dma = c->device;
+        struct ioatdma_device *device = to_ioatdma_device(dma);
+
+        return sprintf(page, "%d.%d\n",
+                       device->version >> 4, device->version & 0xf);
+}
+struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
+
+static struct attribute *ioat1_attrs[] = {
+        &ring_size_attr.attr,
+        &ring_active_attr.attr,
+        &ioat_cap_attr.attr,
+        &ioat_version_attr.attr,
+        NULL,
+};
+
+static ssize_t
+ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
+{
+        struct ioat_sysfs_entry *entry;
+        struct ioat_chan_common *chan;
+
+        entry = container_of(attr, struct ioat_sysfs_entry, attr);
+        chan = container_of(kobj, struct ioat_chan_common, kobj);
+
+        if (!entry->show)
+                return -EIO;
+        return entry->show(&chan->common, page);
+}
+
+struct sysfs_ops ioat_sysfs_ops = {
+        .show   = ioat_attr_show,
+};
+
+static struct kobj_type ioat1_ktype = {
+        .sysfs_ops = &ioat_sysfs_ops,
+        .default_attrs = ioat1_attrs,
+};
+
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
+{
+        struct dma_device *dma = &device->common;
+        struct dma_chan *c;
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                struct ioat_chan_common *chan = to_chan_common(c);
+                struct kobject *parent = &c->dev->device.kobj;
+                int err;
+
+                err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
+                if (err) {
+                        dev_warn(to_dev(chan),
+                                 "sysfs init error (%d), continuing...\n", err);
+                        kobject_put(&chan->kobj);
+                        set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
+                }
+        }
+}
+
+void ioat_kobject_del(struct ioatdma_device *device)
+{
+        struct dma_device *dma = &device->common;
+        struct dma_chan *c;
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                struct ioat_chan_common *chan = to_chan_common(c);
+
+                if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
+                        kobject_del(&chan->kobj);
+                        kobject_put(&chan->kobj);
+                }
+        }
+}
+
+int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
+{
+        struct pci_dev *pdev = device->pdev;
+        struct dma_device *dma;
+        int err;
+
+        device->intr_quirk = ioat1_intr_quirk;
+        device->enumerate_channels = ioat1_enumerate_channels;
+        device->self_test = ioat_dma_self_test;
+        dma = &device->common;
+        dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
+        dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
+        dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
+        dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
+        dma->device_is_tx_complete = ioat1_dma_is_complete;
+
+        err = ioat_probe(device);
+        if (err)
+                return err;
+        ioat_set_tcp_copy_break(4096);
+        err = ioat_register(device);
+        if (err)
+                return err;
+        ioat_kobject_add(device, &ioat1_ktype);
+
+        if (dca)
+                device->dca = ioat_dca_init(pdev, device->reg_base);
+
+        return err;
+}
+
+void __devexit ioat_dma_remove(struct ioatdma_device *device)
+{
+        struct dma_device *dma = &device->common;
+
+        ioat_disable_interrupts(device);
+
+        ioat_kobject_del(device);
+
+        dma_async_device_unregister(dma);
+
+        pci_pool_destroy(device->dma_pool);
+        pci_pool_destroy(device->completion_pool);
+
+        INIT_LIST_HEAD(&dma->channels);
+}
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
new file mode 100644
index 000000000000..c14fdfeb7f33
--- /dev/null
+++ b/drivers/dma/ioat/dma.h
@@ -0,0 +1,337 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef IOATDMA_H
+#define IOATDMA_H
+
+#include <linux/dmaengine.h>
+#include "hw.h"
+#include "registers.h"
+#include <linux/init.h>
+#include <linux/dmapool.h>
+#include <linux/cache.h>
+#include <linux/pci_ids.h>
+#include <net/tcp.h>
+
+#define IOAT_DMA_VERSION  "4.00"
+
+#define IOAT_LOW_COMPLETION_MASK        0xffffffc0
+#define IOAT_DMA_DCA_ANY_CPU            ~0
+
+#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
+#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
+#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd)
+#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev)
+
+#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
+
+/*
+ * workaround for IOAT ver.3.0 null descriptor issue
+ * (channel returns error when size is 0)
+ */
+#define NULL_DESC_BUFFER_SIZE 1
+
+/**
+ * struct ioatdma_device - internal representation of a IOAT device
+ * @pdev: PCI-Express device
+ * @reg_base: MMIO register space base address
+ * @dma_pool: for allocating DMA descriptors
+ * @common: embedded struct dma_device
+ * @version: version of ioatdma device
+ * @msix_entries: irq handlers
+ * @idx: per channel data
+ * @dca: direct cache access context
+ * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
+ * @enumerate_channels: hw version specific channel enumeration
+ * @cleanup_tasklet: select between the v2 and v3 cleanup routines
+ * @timer_fn: select between the v2 and v3 timer watchdog routines
+ * @self_test: hardware version specific self test for each supported op type
+ *
+ * Note: the v3 cleanup routine supports raid operations
+ */
+struct ioatdma_device {
+        struct pci_dev *pdev;
+        void __iomem *reg_base;
+        struct pci_pool *dma_pool;
+        struct pci_pool *completion_pool;
+        struct dma_device common;
+        u8 version;
+        struct msix_entry msix_entries[4];
+        struct ioat_chan_common *idx[4];
+        struct dca_provider *dca;
+        void (*intr_quirk)(struct ioatdma_device *device);
+        int (*enumerate_channels)(struct ioatdma_device *device);
+        void (*cleanup_tasklet)(unsigned long data);
+        void (*timer_fn)(unsigned long data);
+        int (*self_test)(struct ioatdma_device *device);
+};
+
+struct ioat_chan_common {
+        struct dma_chan common;
+        void __iomem *reg_base;
+        unsigned long last_completion;
+        spinlock_t cleanup_lock;
+        dma_cookie_t completed_cookie;
+        unsigned long state;
+        #define IOAT_COMPLETION_PENDING 0
+        #define IOAT_COMPLETION_ACK 1
+        #define IOAT_RESET_PENDING 2
+        #define IOAT_KOBJ_INIT_FAIL 3
+        struct timer_list timer;
+        #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
+        #define IDLE_TIMEOUT msecs_to_jiffies(2000)
+        #define RESET_DELAY msecs_to_jiffies(100)
+        struct ioatdma_device *device;
+        dma_addr_t completion_dma;
+        u64 *completion;
+        struct tasklet_struct cleanup_task;
+        struct kobject kobj;
+};
+
+struct ioat_sysfs_entry {
+        struct attribute attr;
+        ssize_t (*show)(struct dma_chan *, char *);
+};
+
+/**
+ * struct ioat_dma_chan - internal representation of a DMA channel
+ */
+struct ioat_dma_chan {
+        struct ioat_chan_common base;
+
+        size_t xfercap; /* XFERCAP register value expanded out */
+
+        spinlock_t desc_lock;
+        struct list_head free_desc;
+        struct list_head used_desc;
+
+        int pending;
+        u16 desccount;
+        u16 active;
+};
+
+static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
+{
+        return container_of(c, struct ioat_chan_common, common);
+}
+
+static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
+{
+        struct ioat_chan_common *chan = to_chan_common(c);
+
+        return container_of(chan, struct ioat_dma_chan, base);
+}
+
+/**
+ * ioat_is_complete - poll the status of an ioat transaction
+ * @c: channel handle
+ * @cookie: transaction identifier
+ * @done: if set, updated with last completed transaction
+ * @used: if set, updated with last used transaction
+ */
+static inline enum dma_status
+ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                 dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat_chan_common *chan = to_chan_common(c);
+        dma_cookie_t last_used;
+        dma_cookie_t last_complete;
+
+        last_used = c->cookie;
+        last_complete = chan->completed_cookie;
+
+        if (done)
+                *done = last_complete;
+        if (used)
+                *used = last_used;
+
+        return dma_async_is_complete(cookie, last_complete, last_used);
+}
+
+/* wrapper around hardware descriptor format + additional software fields */
+
+/**
+ * struct ioat_desc_sw - wrapper around hardware descriptor
+ * @hw: hardware DMA descriptor (for memcpy)
+ * @node: this descriptor will either be on the free list,
+ *     or attached to a transaction list (tx_list)
+ * @txd: the generic software descriptor for all engines
+ * @id: identifier for debug
+ */
+struct ioat_desc_sw {
+        struct ioat_dma_descriptor *hw;
+        struct list_head node;
+        size_t len;
+        struct list_head tx_list;
+        struct dma_async_tx_descriptor txd;
+        #ifdef DEBUG
+        int id;
+        #endif
+};
+
+#ifdef DEBUG
+#define set_desc_id(desc, i) ((desc)->id = (i))
+#define desc_id(desc) ((desc)->id)
+#else
+#define set_desc_id(desc, i)
+#define desc_id(desc) (0)
+#endif
+
+static inline void
+__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
+                struct dma_async_tx_descriptor *tx, int id)
+{
+        struct device *dev = to_dev(chan);
+
+        dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
+                " ctl: %#x (op: %d int_en: %d compl: %d)\n", id,
+                (unsigned long long) tx->phys,
+                (unsigned long long) hw->next, tx->cookie, tx->flags,
+                hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
+}
+
+#define dump_desc_dbg(c, d) \
+        ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; })
+
+static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
+{
+        #ifdef CONFIG_NET_DMA
+        sysctl_tcp_dma_copybreak = copybreak;
+        #endif
+}
+
+static inline struct ioat_chan_common *
+ioat_chan_by_index(struct ioatdma_device *device, int index)
+{
+        return device->idx[index];
+}
+
+static inline u64 ioat_chansts(struct ioat_chan_common *chan)
+{
+        u8 ver = chan->device->version;
+        u64 status;
+        u32 status_lo;
+
+        /* We need to read the low address first as this causes the
+         * chipset to latch the upper bits for the subsequent read
+         */
+        status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
+        status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
+        status <<= 32;
+        status |= status_lo;
+
+        return status;
+}
+
+static inline void ioat_start(struct ioat_chan_common *chan)
+{
+        u8 ver = chan->device->version;
+
+        writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline u64 ioat_chansts_to_addr(u64 status)
+{
+        return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+}
+
+static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
+{
+        return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+}
+
+static inline void ioat_suspend(struct ioat_chan_common *chan)
+{
+        u8 ver = chan->device->version;
+
+        writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        writel(addr & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
+        writel(addr >> 32,
+               chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
+}
+
+static inline bool is_ioat_active(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
+}
+
+static inline bool is_ioat_idle(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
+}
+
+static inline bool is_ioat_halted(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
+}
+
+static inline bool is_ioat_suspended(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
+}
+
+/* channel was fatally programmed */
+static inline bool is_ioat_bug(unsigned long err)
+{
+        return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
+                         IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
+                         IOAT_CHANERR_LENGTH_ERR));
+}
+
+static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
+                              int direction, enum dma_ctrl_flags flags, bool dst)
+{
+        if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
+            (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
+                pci_unmap_single(pdev, addr, len, direction);
+        else
+                pci_unmap_page(pdev, addr, len, direction);
+}
+
+int __devinit ioat_probe(struct ioatdma_device *device);
+int __devinit ioat_register(struct ioatdma_device *device);
+int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat_dma_self_test(struct ioatdma_device *device);
+void __devexit ioat_dma_remove(struct ioatdma_device *device);
+struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
+                                              void __iomem *iobase);
+unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
+void ioat_init_channel(struct ioatdma_device *device,
+                       struct ioat_chan_common *chan, int idx,
+                       void (*timer_fn)(unsigned long),
+                       void (*tasklet)(unsigned long),
+                       unsigned long ioat);
+void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
+                    size_t len, struct ioat_dma_descriptor *hw);
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+                           unsigned long *phys_complete);
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
+void ioat_kobject_del(struct ioatdma_device *device);
+extern struct sysfs_ops ioat_sysfs_ops;
+extern struct ioat_sysfs_entry ioat_version_attr;
+extern struct ioat_sysfs_entry ioat_cap_attr;
+#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
new file mode 100644
index 000000000000..96ffab7d37a7
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.c
@@ -0,0 +1,871 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2004 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
+ * does asynchronous data movement and checksumming operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/workqueue.h>
+#include <linux/i7300_idle.h>
+#include "dma.h"
+#include "dma_v2.h"
+#include "registers.h"
+#include "hw.h"
+
+int ioat_ring_alloc_order = 8;
+module_param(ioat_ring_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_alloc_order,
+                 "ioat2+: allocate 2^n descriptors per channel"
+                 " (default: 8 max: 16)");
+static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
+module_param(ioat_ring_max_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_max_alloc_order,
+                 "ioat2+: upper limit for ring size (default: 16)");
+
+void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
+{
+        void * __iomem reg_base = ioat->base.reg_base;
+
+        ioat->pending = 0;
+        ioat->dmacount += ioat2_ring_pending(ioat);
+        ioat->issued = ioat->head;
+        /* make descriptor updates globally visible before notifying channel */
+        wmb();
+        writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
+        dev_dbg(to_dev(&ioat->base),
+                "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+}
+
+void ioat2_issue_pending(struct dma_chan *chan)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
+
+        spin_lock_bh(&ioat->ring_lock);
+        if (ioat->pending == 1)
+                __ioat2_issue_pending(ioat);
+        spin_unlock_bh(&ioat->ring_lock);
+}
+
+/**
+ * ioat2_update_pending - log pending descriptors
+ * @ioat: ioat2+ channel
+ *
+ * set pending to '1' unless pending is already set to '2', pending == 2
+ * indicates that submission is temporarily blocked due to an in-flight
+ * reset.  If we are already above the ioat_pending_level threshold then
+ * just issue pending.
+ *
+ * called with ring_lock held
+ */
+static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
+{
+        if (unlikely(ioat->pending == 2))
+                return;
+        else if (ioat2_ring_pending(ioat) > ioat_pending_level)
+                __ioat2_issue_pending(ioat);
+        else
+                ioat->pending = 1;
+}
+
+static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_ring_ent *desc;
+        struct ioat_dma_descriptor *hw;
+        int idx;
+
+        if (ioat2_ring_space(ioat) < 1) {
+                dev_err(to_dev(&ioat->base),
+                        "Unable to start null desc - ring full\n");
+                return;
+        }
+
+        dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued);
+        idx = ioat2_desc_alloc(ioat, 1);
+        desc = ioat2_get_ring_ent(ioat, idx);
+
+        hw = desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = 1;
+        hw->ctl_f.compl_write = 1;
+        /* set size to non-zero value (channel returns error when size is 0) */
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        hw->src_addr = 0;
+        hw->dst_addr = 0;
+        async_tx_ack(&desc->txd);
+        ioat2_set_chainaddr(ioat, desc->txd.phys);
+        dump_desc_dbg(ioat, desc);
+        __ioat2_issue_pending(ioat);
+}
+
+static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
+{
+        spin_lock_bh(&ioat->ring_lock);
+        __ioat2_start_null_desc(ioat);
+        spin_unlock_bh(&ioat->ring_lock);
+}
+
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct dma_async_tx_descriptor *tx;
+        struct ioat_ring_ent *desc;
+        bool seen_current = false;
+        u16 active;
+        int i;
+
+        dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued);
+
+        active = ioat2_ring_active(ioat);
+        for (i = 0; i < active && !seen_current; i++) {
+                prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+                desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+                tx = &desc->txd;
+                dump_desc_dbg(ioat, desc);
+                if (tx->cookie) {
+                        ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+                        chan->completed_cookie = tx->cookie;
+                        tx->cookie = 0;
+                        if (tx->callback) {
+                                tx->callback(tx->callback_param);
+                                tx->callback = NULL;
+                        }
+                }
+
+                if (tx->phys == phys_complete)
+                        seen_current = true;
+        }
+        ioat->tail += i;
+        BUG_ON(!seen_current); /* no active descs have written a completion? */
+
+        chan->last_completion = phys_complete;
+        if (ioat->head == ioat->tail) {
+                dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+                        __func__);
+                clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+}
+
+/**
+ * ioat2_cleanup - clean finished descriptors (advance tail pointer)
+ * @chan: ioat channel to be cleaned up
+ */
+static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        if (!spin_trylock_bh(&chan->cleanup_lock))
+                return;
+
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        if (!spin_trylock_bh(&ioat->ring_lock)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->ring_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+void ioat2_cleanup_tasklet(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+
+        ioat2_cleanup(ioat);
+        writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+void __ioat2_restart_chan(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        /* set the tail to be re-issued */
+        ioat->issued = ioat->tail;
+        ioat->dmacount = 0;
+        set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+        dev_dbg(to_dev(chan),
+                "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+
+        if (ioat2_ring_pending(ioat)) {
+                struct ioat_ring_ent *desc;
+
+                desc = ioat2_get_ring_ent(ioat, ioat->tail);
+                ioat2_set_chainaddr(ioat, desc->txd.phys);
+                __ioat2_issue_pending(ioat);
+        } else
+                __ioat2_start_null_desc(ioat);
+}
+
+static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+        u32 status;
+
+        status = ioat_chansts(chan);
+        if (is_ioat_active(status) || is_ioat_idle(status))
+                ioat_suspend(chan);
+        while (is_ioat_active(status) || is_ioat_idle(status)) {
+                status = ioat_chansts(chan);
+                cpu_relax();
+        }
+
+        if (ioat_cleanup_preamble(chan, &phys_complete))
+                __cleanup(ioat, phys_complete);
+
+        __ioat2_restart_chan(ioat);
+}
+
+void ioat2_timer_event(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat_chan_common *chan = &ioat->base;
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+                unsigned long phys_complete;
+                u64 status;
+
+                spin_lock_bh(&ioat->ring_lock);
+                status = ioat_chansts(chan);
+
+                /* when halted due to errors check for channel
+                 * programming errors before advancing the completion state
+                 */
+                if (is_ioat_halted(status)) {
+                        u32 chanerr;
+
+                        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                        BUG_ON(is_ioat_bug(chanerr));
+                }
+
+                /* if we haven't made progress and we have already
+                 * acknowledged a pending completion once, then be more
+                 * forceful with a restart
+                 */
+                if (ioat_cleanup_preamble(chan, &phys_complete))
+                        __cleanup(ioat, phys_complete);
+                else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                        ioat2_restart_channel(ioat);
+                else {
+                        set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                }
+                spin_unlock_bh(&ioat->ring_lock);
+        } else {
+                u16 active;
+
+                /* if the ring is idle, empty, and oversized try to step
+                 * down the size
+                 */
+                spin_lock_bh(&ioat->ring_lock);
+                active = ioat2_ring_active(ioat);
+                if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+                        reshape_ring(ioat, ioat->alloc_order-1);
+                spin_unlock_bh(&ioat->ring_lock);
+
+                /* keep shrinking until we get back to our minimum
+                 * default size
+                 */
+                if (ioat->alloc_order > ioat_get_alloc_order())
+                        mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+/**
+ * ioat2_enumerate_channels - find and initialize the device's channels
+ * @device: the device to be enumerated
+ */
+int ioat2_enumerate_channels(struct ioatdma_device *device)
+{
+        struct ioat2_dma_chan *ioat;
+        struct device *dev = &device->pdev->dev;
+        struct dma_device *dma = &device->common;
+        u8 xfercap_log;
+        int i;
+
+        INIT_LIST_HEAD(&dma->channels);
+        dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+        dma->chancnt &= 0x1f; /* bits [4:0] valid */
+        if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+                dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+                         dma->chancnt, ARRAY_SIZE(device->idx));
+                dma->chancnt = ARRAY_SIZE(device->idx);
+        }
+        xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+        xfercap_log &= 0x1f; /* bits [4:0] valid */
+        if (xfercap_log == 0)
+                return 0;
+        dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
+
+        /* FIXME which i/oat version is i7300? */
+#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
+        if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+                dma->chancnt--;
+#endif
+        for (i = 0; i < dma->chancnt; i++) {
+                ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
+                if (!ioat)
+                        break;
+
+                ioat_init_channel(device, &ioat->base, i,
+                                  device->timer_fn,
+                                  device->cleanup_tasklet,
+                                  (unsigned long) ioat);
+                ioat->xfercap_log = xfercap_log;
+                spin_lock_init(&ioat->ring_lock);
+        }
+        dma->chancnt = i;
+        return i;
+}
+
+static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
+{
+        struct dma_chan *c = tx->chan;
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        dma_cookie_t cookie = c->cookie;
+
+        cookie++;
+        if (cookie < 0)
+                cookie = 1;
+        tx->cookie = cookie;
+        c->cookie = cookie;
+        dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+
+        if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+                mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+        ioat2_update_pending(ioat);
+        spin_unlock_bh(&ioat->ring_lock);
+
+        return cookie;
+}
+
+static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
+{
+        struct ioat_dma_descriptor *hw;
+        struct ioat_ring_ent *desc;
+        struct ioatdma_device *dma;
+        dma_addr_t phys;
+
+        dma = to_ioatdma_device(chan->device);
+        hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
+        if (!hw)
+                return NULL;
+        memset(hw, 0, sizeof(*hw));
+
+        desc = kmem_cache_alloc(ioat2_cache, flags);
+        if (!desc) {
+                pci_pool_free(dma->dma_pool, hw, phys);
+                return NULL;
+        }
+        memset(desc, 0, sizeof(*desc));
+
+        dma_async_tx_descriptor_init(&desc->txd, chan);
+        desc->txd.tx_submit = ioat2_tx_submit_unlock;
+        desc->hw = hw;
+        desc->txd.phys = phys;
+        return desc;
+}
+
+static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
+{
+        struct ioatdma_device *dma;
+
+        dma = to_ioatdma_device(chan->device);
+        pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
+        kmem_cache_free(ioat2_cache, desc);
+}
+
+static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
+{
+        struct ioat_ring_ent **ring;
+        int descs = 1 << order;
+        int i;
+
+        if (order > ioat_get_max_alloc_order())
+                return NULL;
+
+        /* allocate the array to hold the software ring */
+        ring = kcalloc(descs, sizeof(*ring), flags);
+        if (!ring)
+                return NULL;
+        for (i = 0; i < descs; i++) {
+                ring[i] = ioat2_alloc_ring_ent(c, flags);
+                if (!ring[i]) {
+                        while (i--)
+                                ioat2_free_ring_ent(ring[i], c);
+                        kfree(ring);
+                        return NULL;
+                }
+                set_desc_id(ring[i], i);
+        }
+
+        /* link descs */
+        for (i = 0; i < descs-1; i++) {
+                struct ioat_ring_ent *next = ring[i+1];
+                struct ioat_dma_descriptor *hw = ring[i]->hw;
+
+                hw->next = next->txd.phys;
+        }
+        ring[i]->hw->next = ring[0]->txd.phys;
+
+        return ring;
+}
+
+/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
+ * @chan: channel to be initialized
+ */
+int ioat2_alloc_chan_resources(struct dma_chan *c)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_ring_ent **ring;
+        u32 chanerr;
+        int order;
+
+        /* have we already been set up? */
+        if (ioat->ring)
+                return 1 << ioat->alloc_order;
+
+        /* Setup register to interrupt and write completion status on error */
+        writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+
+        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+        if (chanerr) {
+                dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
+                writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+        }
+
+        /* allocate a completion writeback area */
+        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
+        chan->completion = pci_pool_alloc(chan->device->completion_pool,
+                                          GFP_KERNEL, &chan->completion_dma);
+        if (!chan->completion)
+                return -ENOMEM;
+
+        memset(chan->completion, 0, sizeof(*chan->completion));
+        writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+        writel(((u64) chan->completion_dma) >> 32,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
+        order = ioat_get_alloc_order();
+        ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
+        if (!ring)
+                return -ENOMEM;
+
+        spin_lock_bh(&ioat->ring_lock);
+        ioat->ring = ring;
+        ioat->head = 0;
+        ioat->issued = 0;
+        ioat->tail = 0;
+        ioat->pending = 0;
+        ioat->alloc_order = order;
+        spin_unlock_bh(&ioat->ring_lock);
+
+        tasklet_enable(&chan->cleanup_task);
+        ioat2_start_null_desc(ioat);
+
+        return 1 << ioat->alloc_order;
+}
+
+bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
+{
+        /* reshape differs from normal ring allocation in that we want
+         * to allocate a new software ring while only
+         * extending/truncating the hardware ring
+         */
+        struct ioat_chan_common *chan = &ioat->base;
+        struct dma_chan *c = &chan->common;
+        const u16 curr_size = ioat2_ring_mask(ioat) + 1;
+        const u16 active = ioat2_ring_active(ioat);
+        const u16 new_size = 1 << order;
+        struct ioat_ring_ent **ring;
+        u16 i;
+
+        if (order > ioat_get_max_alloc_order())
+                return false;
+
+        /* double check that we have at least 1 free descriptor */
+        if (active == curr_size)
+                return false;
+
+        /* when shrinking, verify that we can hold the current active
+         * set in the new ring
+         */
+        if (active >= new_size)
+                return false;
+
+        /* allocate the array to hold the software ring */
+        ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
+        if (!ring)
+                return false;
+
+        /* allocate/trim descriptors as needed */
+        if (new_size > curr_size) {
+                /* copy current descriptors to the new ring */
+                for (i = 0; i < curr_size; i++) {
+                        u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                        ring[new_idx] = ioat->ring[curr_idx];
+                        set_desc_id(ring[new_idx], new_idx);
+                }
+
+                /* add new descriptors to the ring */
+                for (i = curr_size; i < new_size; i++) {
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                        ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
+                        if (!ring[new_idx]) {
+                                while (i--) {
+                                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                                        ioat2_free_ring_ent(ring[new_idx], c);
+                                }
+                                kfree(ring);
+                                return false;
+                        }
+                        set_desc_id(ring[new_idx], new_idx);
+                }
+
+                /* hw link new descriptors */
+                for (i = curr_size-1; i < new_size; i++) {
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+                        struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
+                        struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
+
+                        hw->next = next->txd.phys;
+                }
+        } else {
+                struct ioat_dma_descriptor *hw;
+                struct ioat_ring_ent *next;
+
+                /* copy current descriptors to the new ring, dropping the
+                 * removed descriptors
+                 */
+                for (i = 0; i < new_size; i++) {
+                        u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                        ring[new_idx] = ioat->ring[curr_idx];
+                        set_desc_id(ring[new_idx], new_idx);
+                }
+
+                /* free deleted descriptors */
+                for (i = new_size; i < curr_size; i++) {
+                        struct ioat_ring_ent *ent;
+
+                        ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
+                        ioat2_free_ring_ent(ent, c);
+                }
+
+                /* fix up hardware ring */
+                hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
+                next = ring[(ioat->tail+new_size) & (new_size-1)];
+                hw->next = next->txd.phys;
+        }
+
+        dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+                __func__, new_size);
+
+        kfree(ioat->ring);
+        ioat->ring = ring;
+        ioat->alloc_order = order;
+
+        return true;
+}
+
+/**
+ * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
+ * @idx: gets starting descriptor index on successful allocation
+ * @ioat: ioat2,3 channel (ring) to operate on
+ * @num_descs: allocation length
+ */
+int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        spin_lock_bh(&ioat->ring_lock);
+        /* never allow the last descriptor to be consumed, we need at
+         * least one free at all times to allow for on-the-fly ring
+         * resizing.
+         */
+        while (unlikely(ioat2_ring_space(ioat) <= num_descs)) {
+                if (reshape_ring(ioat, ioat->alloc_order + 1) &&
+                    ioat2_ring_space(ioat) > num_descs)
+                                break;
+
+                if (printk_ratelimit())
+                        dev_dbg(to_dev(chan),
+                                "%s: ring full! num_descs: %d (%x:%x:%x)\n",
+                                __func__, num_descs, ioat->head, ioat->tail,
+                                ioat->issued);
+                spin_unlock_bh(&ioat->ring_lock);
+
+                /* progress reclaim in the allocation failure case we
+                 * may be called under bh_disabled so we need to trigger
+                 * the timer event directly
+                 */
+                spin_lock_bh(&chan->cleanup_lock);
+                if (jiffies > chan->timer.expires &&
+                    timer_pending(&chan->timer)) {
+                        struct ioatdma_device *device = chan->device;
+
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                        spin_unlock_bh(&chan->cleanup_lock);
+                        device->timer_fn((unsigned long) ioat);
+                } else
+                        spin_unlock_bh(&chan->cleanup_lock);
+                return -ENOMEM;
+        }
+
+        dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
+                __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
+
+        *idx = ioat2_desc_alloc(ioat, num_descs);
+        return 0;  /* with ioat->ring_lock held */
+}
+
+struct dma_async_tx_descriptor *
+ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
+                           dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_dma_descriptor *hw;
+        struct ioat_ring_ent *desc;
+        dma_addr_t dst = dma_dest;
+        dma_addr_t src = dma_src;
+        size_t total_len = len;
+        int num_descs;
+        u16 idx;
+        int i;
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                hw = desc->hw;
+
+                hw->size = copy;
+                hw->ctl = 0;
+                hw->src_addr = src;
+                hw->dst_addr = dst;
+
+                len -= copy;
+                dst += copy;
+                src += copy;
+                dump_desc_dbg(ioat, desc);
+        } while (++i < num_descs);
+
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        hw->ctl_f.compl_write = 1;
+        dump_desc_dbg(ioat, desc);
+        /* we leave the channel locked to ensure in order submission */
+
+        return &desc->txd;
+}
+
+/**
+ * ioat2_free_chan_resources - release all the descriptors
+ * @chan: the channel to be cleaned
+ */
+void ioat2_free_chan_resources(struct dma_chan *c)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioatdma_device *device = chan->device;
+        struct ioat_ring_ent *desc;
+        const u16 total_descs = 1 << ioat->alloc_order;
+        int descs;
+        int i;
+
+        /* Before freeing channel resources first check
+         * if they have been previously allocated for this channel.
+         */
+        if (!ioat->ring)
+                return;
+
+        tasklet_disable(&chan->cleanup_task);
+        del_timer_sync(&chan->timer);
+        device->cleanup_tasklet((unsigned long) ioat);
+
+        /* Delay 100ms after reset to allow internal DMA logic to quiesce
+         * before removing DMA descriptor resources.
+         */
+        writeb(IOAT_CHANCMD_RESET,
+               chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+        mdelay(100);
+
+        spin_lock_bh(&ioat->ring_lock);
+        descs = ioat2_ring_space(ioat);
+        dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
+        for (i = 0; i < descs; i++) {
+                desc = ioat2_get_ring_ent(ioat, ioat->head + i);
+                ioat2_free_ring_ent(desc, c);
+        }
+
+        if (descs < total_descs)
+                dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
+                        total_descs - descs);
+
+        for (i = 0; i < total_descs - descs; i++) {
+                desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+                dump_desc_dbg(ioat, desc);
+                ioat2_free_ring_ent(desc, c);
+        }
+
+        kfree(ioat->ring);
+        ioat->ring = NULL;
+        ioat->alloc_order = 0;
+        pci_pool_free(device->completion_pool, chan->completion,
+                      chan->completion_dma);
+        spin_unlock_bh(&ioat->ring_lock);
+
+        chan->last_completion = 0;
+        chan->completion_dma = 0;
+        ioat->pending = 0;
+        ioat->dmacount = 0;
+}
+
+enum dma_status
+ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                     dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioatdma_device *device = ioat->base.device;
+
+        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+                return DMA_SUCCESS;
+
+        device->cleanup_tasklet((unsigned long) ioat);
+
+        return ioat_is_complete(c, cookie, done, used);
+}
+
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+        return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+        /* ...taken outside the lock, no need to be precise */
+        return sprintf(page, "%d\n", ioat2_ring_active(ioat));
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static struct attribute *ioat2_attrs[] = {
+        &ring_size_attr.attr,
+        &ring_active_attr.attr,
+        &ioat_cap_attr.attr,
+        &ioat_version_attr.attr,
+        NULL,
+};
+
+struct kobj_type ioat2_ktype = {
+        .sysfs_ops = &ioat_sysfs_ops,
+        .default_attrs = ioat2_attrs,
+};
+
+int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
+{
+        struct pci_dev *pdev = device->pdev;
+        struct dma_device *dma;
+        struct dma_chan *c;
+        struct ioat_chan_common *chan;
+        int err;
+
+        device->enumerate_channels = ioat2_enumerate_channels;
+        device->cleanup_tasklet = ioat2_cleanup_tasklet;
+        device->timer_fn = ioat2_timer_event;
+        device->self_test = ioat_dma_self_test;
+        dma = &device->common;
+        dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
+        dma->device_issue_pending = ioat2_issue_pending;
+        dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
+        dma->device_free_chan_resources = ioat2_free_chan_resources;
+        dma->device_is_tx_complete = ioat2_is_complete;
+
+        err = ioat_probe(device);
+        if (err)
+                return err;
+        ioat_set_tcp_copy_break(2048);
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                chan = to_chan_common(c);
+                writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
+                       chan->reg_base + IOAT_DCACTRL_OFFSET);
+        }
+
+        err = ioat_register(device);
+        if (err)
+                return err;
+
+        ioat_kobject_add(device, &ioat2_ktype);
+
+        if (dca)
+                device->dca = ioat2_dca_init(pdev, device->reg_base);
+
+        return err;
+}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
new file mode 100644
index 000000000000..1d849ef74d5f
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.h
@@ -0,0 +1,190 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef IOATDMA_V2_H
+#define IOATDMA_V2_H
+
+#include <linux/dmaengine.h>
+#include "dma.h"
+#include "hw.h"
+
+
+extern int ioat_pending_level;
+extern int ioat_ring_alloc_order;
+
+/*
+ * workaround for IOAT ver.3.0 null descriptor issue
+ * (channel returns error when size is 0)
+ */
+#define NULL_DESC_BUFFER_SIZE 1
+
+#define IOAT_MAX_ORDER 16
+#define ioat_get_alloc_order() \
+        (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
+#define ioat_get_max_alloc_order() \
+        (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
+
+/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
+ * @base: common ioat channel parameters
+ * @xfercap_log; log2 of channel max transfer length (for fast division)
+ * @head: allocated index
+ * @issued: hardware notification point
+ * @tail: cleanup index
+ * @pending: lock free indicator for issued != head
+ * @dmacount: identical to 'head' except for occasionally resetting to zero
+ * @alloc_order: log2 of the number of allocated descriptors
+ * @ring: software ring buffer implementation of hardware ring
+ * @ring_lock: protects ring attributes
+ */
+struct ioat2_dma_chan {
+        struct ioat_chan_common base;
+        size_t xfercap_log;
+        u16 head;
+        u16 issued;
+        u16 tail;
+        u16 dmacount;
+        u16 alloc_order;
+        int pending;
+        struct ioat_ring_ent **ring;
+        spinlock_t ring_lock;
+};
+
+static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
+{
+        struct ioat_chan_common *chan = to_chan_common(c);
+
+        return container_of(chan, struct ioat2_dma_chan, base);
+}
+
+static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat)
+{
+        return (1 << ioat->alloc_order) - 1;
+}
+
+/* count of descriptors in flight with the engine */
+static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
+{
+        return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat);
+}
+
+/* count of descriptors pending submission to hardware */
+static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
+{
+        return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat);
+}
+
+static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat)
+{
+        u16 num_descs = ioat2_ring_mask(ioat) + 1;
+        u16 active = ioat2_ring_active(ioat);
+
+        BUG_ON(active > num_descs);
+
+        return num_descs - active;
+}
+
+/* assumes caller already checked space */
+static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len)
+{
+        ioat->head += len;
+        return ioat->head - len;
+}
+
+static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
+{
+        u16 num_descs = len >> ioat->xfercap_log;
+
+        num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
+        return num_descs;
+}
+
+/**
+ * struct ioat_ring_ent - wrapper around hardware descriptor
+ * @hw: hardware DMA descriptor (for memcpy)
+ * @fill: hardware fill descriptor
+ * @xor: hardware xor descriptor
+ * @xor_ex: hardware xor extension descriptor
+ * @pq: hardware pq descriptor
+ * @pq_ex: hardware pq extension descriptor
+ * @pqu: hardware pq update descriptor
+ * @raw: hardware raw (un-typed) descriptor
+ * @txd: the generic software descriptor for all engines
+ * @len: total transaction length for unmap
+ * @result: asynchronous result of validate operations
+ * @id: identifier for debug
+ */
+
+struct ioat_ring_ent {
+        union {
+                struct ioat_dma_descriptor *hw;
+                struct ioat_fill_descriptor *fill;
+                struct ioat_xor_descriptor *xor;
+                struct ioat_xor_ext_descriptor *xor_ex;
+                struct ioat_pq_descriptor *pq;
+                struct ioat_pq_ext_descriptor *pq_ex;
+                struct ioat_pq_update_descriptor *pqu;
+                struct ioat_raw_descriptor *raw;
+        };
+        size_t len;
+        struct dma_async_tx_descriptor txd;
+        enum sum_check_flags *result;
+        #ifdef DEBUG
+        int id;
+        #endif
+};
+
+static inline struct ioat_ring_ent *
+ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
+{
+        return ioat->ring[idx & ioat2_ring_mask(ioat)];
+}
+
+static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        writel(addr & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
+        writel(addr >> 32,
+               chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+}
+
+int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
+struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs);
+int ioat2_enumerate_channels(struct ioatdma_device *device);
+struct dma_async_tx_descriptor *
+ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
+                           dma_addr_t dma_src, size_t len, unsigned long flags);
+void ioat2_issue_pending(struct dma_chan *chan);
+int ioat2_alloc_chan_resources(struct dma_chan *c);
+void ioat2_free_chan_resources(struct dma_chan *c);
+enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                                  dma_cookie_t *done, dma_cookie_t *used);
+void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
+bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
+void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
+void ioat2_cleanup_tasklet(unsigned long data);
+void ioat2_timer_event(unsigned long data);
+extern struct kobj_type ioat2_ktype;
+extern struct kmem_cache *ioat2_cache;
+#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
new file mode 100644
index 000000000000..35d1e33afd5b
--- /dev/null
+++ b/drivers/dma/ioat/dma_v3.c
@@ -0,0 +1,1223 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ *   * Neither the name of Intel Corporation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Support routines for v3+ hardware
+ */
+
+#include <linux/pci.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include "registers.h"
+#include "hw.h"
+#include "dma.h"
+#include "dma_v2.h"
+
+/* ioat hardware assumes at least two sources for raid operations */
+#define src_cnt_to_sw(x) ((x) + 2)
+#define src_cnt_to_hw(x) ((x) - 2)
+
+/* provide a lookup table for setting the source address in the base or
+ * extended descriptor of an xor or pq descriptor
+ */
+static const u8 xor_idx_to_desc __read_mostly = 0xd0;
+static const u8 xor_idx_to_field[] __read_mostly = { 1, 4, 5, 6, 7, 0, 1, 2 };
+static const u8 pq_idx_to_desc __read_mostly = 0xf8;
+static const u8 pq_idx_to_field[] __read_mostly = { 1, 4, 5, 0, 1, 2, 4, 5 };
+
+static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
+{
+        struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
+
+        return raw->field[xor_idx_to_field[idx]];
+}
+
+static void xor_set_src(struct ioat_raw_descriptor *descs[2],
+                        dma_addr_t addr, u32 offset, int idx)
+{
+        struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
+
+        raw->field[xor_idx_to_field[idx]] = addr + offset;
+}
+
+static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
+{
+        struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
+
+        return raw->field[pq_idx_to_field[idx]];
+}
+
+static void pq_set_src(struct ioat_raw_descriptor *descs[2],
+                       dma_addr_t addr, u32 offset, u8 coef, int idx)
+{
+        struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
+        struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
+
+        raw->field[pq_idx_to_field[idx]] = addr + offset;
+        pq->coef[idx] = coef;
+}
+
+static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
+                            struct ioat_ring_ent *desc, int idx)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct pci_dev *pdev = chan->device->pdev;
+        size_t len = desc->len;
+        size_t offset = len - desc->hw->size;
+        struct dma_async_tx_descriptor *tx = &desc->txd;
+        enum dma_ctrl_flags flags = tx->flags;
+
+        switch (desc->hw->ctl_f.op) {
+        case IOAT_OP_COPY:
+                if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
+                        ioat_dma_unmap(chan, flags, len, desc->hw);
+                break;
+        case IOAT_OP_FILL: {
+                struct ioat_fill_descriptor *hw = desc->fill;
+
+                if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                        ioat_unmap(pdev, hw->dst_addr - offset, len,
+                                   PCI_DMA_FROMDEVICE, flags, 1);
+                break;
+        }
+        case IOAT_OP_XOR_VAL:
+        case IOAT_OP_XOR: {
+                struct ioat_xor_descriptor *xor = desc->xor;
+                struct ioat_ring_ent *ext;
+                struct ioat_xor_ext_descriptor *xor_ex = NULL;
+                int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
+                struct ioat_raw_descriptor *descs[2];
+                int i;
+
+                if (src_cnt > 5) {
+                        ext = ioat2_get_ring_ent(ioat, idx + 1);
+                        xor_ex = ext->xor_ex;
+                }
+
+                if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        descs[0] = (struct ioat_raw_descriptor *) xor;
+                        descs[1] = (struct ioat_raw_descriptor *) xor_ex;
+                        for (i = 0; i < src_cnt; i++) {
+                                dma_addr_t src = xor_get_src(descs, i);
+
+                                ioat_unmap(pdev, src - offset, len,
+                                           PCI_DMA_TODEVICE, flags, 0);
+                        }
+
+                        /* dest is a source in xor validate operations */
+                        if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
+                                ioat_unmap(pdev, xor->dst_addr - offset, len,
+                                           PCI_DMA_TODEVICE, flags, 1);
+                                break;
+                        }
+                }
+
+                if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                        ioat_unmap(pdev, xor->dst_addr - offset, len,
+                                   PCI_DMA_FROMDEVICE, flags, 1);
+                break;
+        }
+        case IOAT_OP_PQ_VAL:
+        case IOAT_OP_PQ: {
+                struct ioat_pq_descriptor *pq = desc->pq;
+                struct ioat_ring_ent *ext;
+                struct ioat_pq_ext_descriptor *pq_ex = NULL;
+                int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
+                struct ioat_raw_descriptor *descs[2];
+                int i;
+
+                if (src_cnt > 3) {
+                        ext = ioat2_get_ring_ent(ioat, idx + 1);
+                        pq_ex = ext->pq_ex;
+                }
+
+                /* in the 'continue' case don't unmap the dests as sources */
+                if (dmaf_p_disabled_continue(flags))
+                        src_cnt--;
+                else if (dmaf_continue(flags))
+                        src_cnt -= 3;
+
+                if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        descs[0] = (struct ioat_raw_descriptor *) pq;
+                        descs[1] = (struct ioat_raw_descriptor *) pq_ex;
+                        for (i = 0; i < src_cnt; i++) {
+                                dma_addr_t src = pq_get_src(descs, i);
+
+                                ioat_unmap(pdev, src - offset, len,
+                                           PCI_DMA_TODEVICE, flags, 0);
+                        }
+
+                        /* the dests are sources in pq validate operations */
+                        if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
+                                if (!(flags & DMA_PREP_PQ_DISABLE_P))
+                                        ioat_unmap(pdev, pq->p_addr - offset,
+                                                   len, PCI_DMA_TODEVICE, flags, 0);
+                                if (!(flags & DMA_PREP_PQ_DISABLE_Q))
+                                        ioat_unmap(pdev, pq->q_addr - offset,
+                                                   len, PCI_DMA_TODEVICE, flags, 0);
+                                break;
+                        }
+                }
+
+                if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                        if (!(flags & DMA_PREP_PQ_DISABLE_P))
+                                ioat_unmap(pdev, pq->p_addr - offset, len,
+                                           PCI_DMA_BIDIRECTIONAL, flags, 1);
+                        if (!(flags & DMA_PREP_PQ_DISABLE_Q))
+                                ioat_unmap(pdev, pq->q_addr - offset, len,
+                                           PCI_DMA_BIDIRECTIONAL, flags, 1);
+                }
+                break;
+        }
+        default:
+                dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
+                        __func__, desc->hw->ctl_f.op);
+        }
+}
+
+static bool desc_has_ext(struct ioat_ring_ent *desc)
+{
+        struct ioat_dma_descriptor *hw = desc->hw;
+
+        if (hw->ctl_f.op == IOAT_OP_XOR ||
+            hw->ctl_f.op == IOAT_OP_XOR_VAL) {
+                struct ioat_xor_descriptor *xor = desc->xor;
+
+                if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
+                        return true;
+        } else if (hw->ctl_f.op == IOAT_OP_PQ ||
+                   hw->ctl_f.op == IOAT_OP_PQ_VAL) {
+                struct ioat_pq_descriptor *pq = desc->pq;
+
+                if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
+                        return true;
+        }
+
+        return false;
+}
+
+/**
+ * __cleanup - reclaim used descriptors
+ * @ioat: channel (ring) to clean
+ *
+ * The difference from the dma_v2.c __cleanup() is that this routine
+ * handles extended descriptors and dma-unmapping raid operations.
+ */
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_ring_ent *desc;
+        bool seen_current = false;
+        u16 active;
+        int i;
+
+        dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued);
+
+        active = ioat2_ring_active(ioat);
+        for (i = 0; i < active && !seen_current; i++) {
+                struct dma_async_tx_descriptor *tx;
+
+                prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+                desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+                dump_desc_dbg(ioat, desc);
+                tx = &desc->txd;
+                if (tx->cookie) {
+                        chan->completed_cookie = tx->cookie;
+                        ioat3_dma_unmap(ioat, desc, ioat->tail + i);
+                        tx->cookie = 0;
+                        if (tx->callback) {
+                                tx->callback(tx->callback_param);
+                                tx->callback = NULL;
+                        }
+                }
+
+                if (tx->phys == phys_complete)
+                        seen_current = true;
+
+                /* skip extended descriptors */
+                if (desc_has_ext(desc)) {
+                        BUG_ON(i + 1 >= active);
+                        i++;
+                }
+        }
+        ioat->tail += i;
+        BUG_ON(!seen_current); /* no active descs have written a completion? */
+        chan->last_completion = phys_complete;
+        if (ioat->head == ioat->tail) {
+                dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+                        __func__);
+                clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+}
+
+static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        if (!spin_trylock_bh(&chan->cleanup_lock))
+                return;
+
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        if (!spin_trylock_bh(&ioat->ring_lock)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->ring_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat3_cleanup_tasklet(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+
+        ioat3_cleanup(ioat);
+        writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN,
+               ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+        u32 status;
+
+        status = ioat_chansts(chan);
+        if (is_ioat_active(status) || is_ioat_idle(status))
+                ioat_suspend(chan);
+        while (is_ioat_active(status) || is_ioat_idle(status)) {
+                status = ioat_chansts(chan);
+                cpu_relax();
+        }
+
+        if (ioat_cleanup_preamble(chan, &phys_complete))
+                __cleanup(ioat, phys_complete);
+
+        __ioat2_restart_chan(ioat);
+}
+
+static void ioat3_timer_event(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat_chan_common *chan = &ioat->base;
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+                unsigned long phys_complete;
+                u64 status;
+
+                spin_lock_bh(&ioat->ring_lock);
+                status = ioat_chansts(chan);
+
+                /* when halted due to errors check for channel
+                 * programming errors before advancing the completion state
+                 */
+                if (is_ioat_halted(status)) {
+                        u32 chanerr;
+
+                        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                        BUG_ON(is_ioat_bug(chanerr));
+                }
+
+                /* if we haven't made progress and we have already
+                 * acknowledged a pending completion once, then be more
+                 * forceful with a restart
+                 */
+                if (ioat_cleanup_preamble(chan, &phys_complete))
+                        __cleanup(ioat, phys_complete);
+                else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                        ioat3_restart_channel(ioat);
+                else {
+                        set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                }
+                spin_unlock_bh(&ioat->ring_lock);
+        } else {
+                u16 active;
+
+                /* if the ring is idle, empty, and oversized try to step
+                 * down the size
+                 */
+                spin_lock_bh(&ioat->ring_lock);
+                active = ioat2_ring_active(ioat);
+                if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+                        reshape_ring(ioat, ioat->alloc_order-1);
+                spin_unlock_bh(&ioat->ring_lock);
+
+                /* keep shrinking until we get back to our minimum
+                 * default size
+                 */
+                if (ioat->alloc_order > ioat_get_alloc_order())
+                        mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static enum dma_status
+ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                  dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+                return DMA_SUCCESS;
+
+        ioat3_cleanup(ioat);
+
+        return ioat_is_complete(c, cookie, done, used);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value,
+                       size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_ring_ent *desc;
+        size_t total_len = len;
+        struct ioat_fill_descriptor *fill;
+        int num_descs;
+        u64 src_data = (0x0101010101010101ULL) * (value & 0xff);
+        u16 idx;
+        int i;
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                fill = desc->fill;
+
+                fill->size = xfer_size;
+                fill->src_data = src_data;
+                fill->dst_addr = dest;
+                fill->ctl = 0;
+                fill->ctl_f.op = IOAT_OP_FILL;
+
+                len -= xfer_size;
+                dest += xfer_size;
+                dump_desc_dbg(ioat, desc);
+        } while (++i < num_descs);
+
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        fill->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        fill->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        fill->ctl_f.compl_write = 1;
+        dump_desc_dbg(ioat, desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
+                      dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
+                      size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_ring_ent *compl_desc;
+        struct ioat_ring_ent *desc;
+        struct ioat_ring_ent *ext;
+        size_t total_len = len;
+        struct ioat_xor_descriptor *xor;
+        struct ioat_xor_ext_descriptor *xor_ex = NULL;
+        struct ioat_dma_descriptor *hw;
+        u32 offset = 0;
+        int num_descs;
+        int with_ext;
+        int i;
+        u16 idx;
+        u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
+
+        BUG_ON(src_cnt < 2);
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        /* we need 2x the number of descriptors to cover greater than 5
+         * sources
+         */
+        if (src_cnt > 5) {
+                with_ext = 1;
+                num_descs *= 2;
+        } else
+                with_ext = 0;
+
+        /* completion writes from the raid engine may pass completion
+         * writes from the legacy engine, so we need one extra null
+         * (legacy) descriptor to ensure all completion writes arrive in
+         * order.
+         */
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                struct ioat_raw_descriptor *descs[2];
+                size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+                int s;
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                xor = desc->xor;
+
+                /* save a branch by unconditionally retrieving the
+                 * extended descriptor xor_set_src() knows to not write
+                 * to it in the single descriptor case
+                 */
+                ext = ioat2_get_ring_ent(ioat, idx + i + 1);
+                xor_ex = ext->xor_ex;
+
+                descs[0] = (struct ioat_raw_descriptor *) xor;
+                descs[1] = (struct ioat_raw_descriptor *) xor_ex;
+                for (s = 0; s < src_cnt; s++)
+                        xor_set_src(descs, src[s], offset, s);
+                xor->size = xfer_size;
+                xor->dst_addr = dest + offset;
+                xor->ctl = 0;
+                xor->ctl_f.op = op;
+                xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
+
+                len -= xfer_size;
+                offset += xfer_size;
+                dump_desc_dbg(ioat, desc);
+        } while ((i += 1 + with_ext) < num_descs);
+
+        /* last xor descriptor carries the unmap parameters and fence bit */
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        if (result)
+                desc->result = result;
+        xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+
+        /* completion descriptor carries interrupt bit */
+        compl_desc = ioat2_get_ring_ent(ioat, idx + i);
+        compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
+        hw = compl_desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.compl_write = 1;
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        dump_desc_dbg(ioat, compl_desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
+               unsigned int src_cnt, size_t len, unsigned long flags)
+{
+        return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
+                    unsigned int src_cnt, size_t len,
+                    enum sum_check_flags *result, unsigned long flags)
+{
+        /* the cleanup routine only sets bits on validate failure, it
+         * does not clear bits on validate success... so clear it here
+         */
+        *result = 0;
+
+        return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
+                                     src_cnt - 1, len, flags);
+}
+
+static void
+dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)
+{
+        struct device *dev = to_dev(&ioat->base);
+        struct ioat_pq_descriptor *pq = desc->pq;
+        struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
+        struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
+        int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
+        int i;
+
+        dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
+                " sz: %#x ctl: %#x (op: %d int: %d compl: %d pq: '%s%s' src_cnt: %d)\n",
+                desc_id(desc), (unsigned long long) desc->txd.phys,
+                (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
+                desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,
+                pq->ctl_f.compl_write,
+                pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
+                pq->ctl_f.src_cnt);
+        for (i = 0; i < src_cnt; i++)
+                dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
+                        (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
+        dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
+        dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
+}
+
+static struct dma_async_tx_descriptor *
+__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
+                     const dma_addr_t *dst, const dma_addr_t *src,
+                     unsigned int src_cnt, const unsigned char *scf,
+                     size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_ring_ent *compl_desc;
+        struct ioat_ring_ent *desc;
+        struct ioat_ring_ent *ext;
+        size_t total_len = len;
+        struct ioat_pq_descriptor *pq;
+        struct ioat_pq_ext_descriptor *pq_ex = NULL;
+        struct ioat_dma_descriptor *hw;
+        u32 offset = 0;
+        int num_descs;
+        int with_ext;
+        int i, s;
+        u16 idx;
+        u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
+
+        dev_dbg(to_dev(chan), "%s\n", __func__);
+        /* the engine requires at least two sources (we provide
+         * at least 1 implied source in the DMA_PREP_CONTINUE case)
+         */
+        BUG_ON(src_cnt + dmaf_continue(flags) < 2);
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        /* we need 2x the number of descriptors to cover greater than 3
+         * sources
+         */
+        if (src_cnt > 3 || flags & DMA_PREP_CONTINUE) {
+                with_ext = 1;
+                num_descs *= 2;
+        } else
+                with_ext = 0;
+
+        /* completion writes from the raid engine may pass completion
+         * writes from the legacy engine, so we need one extra null
+         * (legacy) descriptor to ensure all completion writes arrive in
+         * order.
+         */
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                struct ioat_raw_descriptor *descs[2];
+                size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                pq = desc->pq;
+
+                /* save a branch by unconditionally retrieving the
+                 * extended descriptor pq_set_src() knows to not write
+                 * to it in the single descriptor case
+                 */
+                ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
+                pq_ex = ext->pq_ex;
+
+                descs[0] = (struct ioat_raw_descriptor *) pq;
+                descs[1] = (struct ioat_raw_descriptor *) pq_ex;
+
+                for (s = 0; s < src_cnt; s++)
+                        pq_set_src(descs, src[s], offset, scf[s], s);
+
+                /* see the comment for dma_maxpq in include/linux/dmaengine.h */
+                if (dmaf_p_disabled_continue(flags))
+                        pq_set_src(descs, dst[1], offset, 1, s++);
+                else if (dmaf_continue(flags)) {
+                        pq_set_src(descs, dst[0], offset, 0, s++);
+                        pq_set_src(descs, dst[1], offset, 1, s++);
+                        pq_set_src(descs, dst[1], offset, 0, s++);
+                }
+                pq->size = xfer_size;
+                pq->p_addr = dst[0] + offset;
+                pq->q_addr = dst[1] + offset;
+                pq->ctl = 0;
+                pq->ctl_f.op = op;
+                pq->ctl_f.src_cnt = src_cnt_to_hw(s);
+                pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
+                pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
+
+                len -= xfer_size;
+                offset += xfer_size;
+        } while ((i += 1 + with_ext) < num_descs);
+
+        /* last pq descriptor carries the unmap parameters and fence bit */
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        if (result)
+                desc->result = result;
+        pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        dump_pq_desc_dbg(ioat, desc, ext);
+
+        /* completion descriptor carries interrupt bit */
+        compl_desc = ioat2_get_ring_ent(ioat, idx + i);
+        compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
+        hw = compl_desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.compl_write = 1;
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        dump_desc_dbg(ioat, compl_desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
+              unsigned int src_cnt, const unsigned char *scf, size_t len,
+              unsigned long flags)
+{
+        /* handle the single source multiply case from the raid6
+         * recovery path
+         */
+        if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) {
+                dma_addr_t single_source[2];
+                unsigned char single_source_coef[2];
+
+                BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
+                single_source[0] = src[0];
+                single_source[1] = src[0];
+                single_source_coef[0] = scf[0];
+                single_source_coef[1] = 0;
+
+                return __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2,
+                                            single_source_coef, len, flags);
+        } else
+                return __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, scf,
+                                            len, flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
+                  unsigned int src_cnt, const unsigned char *scf, size_t len,
+                  enum sum_check_flags *pqres, unsigned long flags)
+{
+        /* the cleanup routine only sets bits on validate failure, it
+         * does not clear bits on validate success... so clear it here
+         */
+        *pqres = 0;
+
+        return __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
+                                    flags);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
+                 unsigned int src_cnt, size_t len, unsigned long flags)
+{
+        unsigned char scf[src_cnt];
+        dma_addr_t pq[2];
+
+        memset(scf, 0, src_cnt);
+        flags |= DMA_PREP_PQ_DISABLE_Q;
+        pq[0] = dst;
+        pq[1] = ~0;
+
+        return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
+                                    flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
+                     unsigned int src_cnt, size_t len,
+                     enum sum_check_flags *result, unsigned long flags)
+{
+        unsigned char scf[src_cnt];
+        dma_addr_t pq[2];
+
+        /* the cleanup routine only sets bits on validate failure, it
+         * does not clear bits on validate success... so clear it here
+         */
+        *result = 0;
+
+        memset(scf, 0, src_cnt);
+        flags |= DMA_PREP_PQ_DISABLE_Q;
+        pq[0] = src[0];
+        pq[1] = ~0;
+
+        return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
+                                    len, flags);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_ring_ent *desc;
+        struct ioat_dma_descriptor *hw;
+        u16 idx;
+
+        if (ioat2_alloc_and_lock(&idx, ioat, 1) == 0)
+                desc = ioat2_get_ring_ent(ioat, idx);
+        else
+                return NULL;
+
+        hw = desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = 1;
+        hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        hw->ctl_f.compl_write = 1;
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        hw->src_addr = 0;
+        hw->dst_addr = 0;
+
+        desc->txd.flags = flags;
+        desc->len = 1;
+
+        dump_desc_dbg(ioat, desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static void __devinit ioat3_dma_test_callback(void *dma_async_param)
+{
+        struct completion *cmp = dma_async_param;
+
+        complete(cmp);
+}
+
+#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
+static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device)
+{
+        int i, src_idx;
+        struct page *dest;
+        struct page *xor_srcs[IOAT_NUM_SRC_TEST];
+        struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
+        dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
+        dma_addr_t dma_addr, dest_dma;
+        struct dma_async_tx_descriptor *tx;
+        struct dma_chan *dma_chan;
+        dma_cookie_t cookie;
+        u8 cmp_byte = 0;
+        u32 cmp_word;
+        u32 xor_val_result;
+        int err = 0;
+        struct completion cmp;
+        unsigned long tmo;
+        struct device *dev = &device->pdev->dev;
+        struct dma_device *dma = &device->common;
+
+        dev_dbg(dev, "%s\n", __func__);
+
+        if (!dma_has_cap(DMA_XOR, dma->cap_mask))
+                return 0;
+
+        for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
+                xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
+                if (!xor_srcs[src_idx]) {
+                        while (src_idx--)
+                                __free_page(xor_srcs[src_idx]);
+                        return -ENOMEM;
+                }
+        }
+
+        dest = alloc_page(GFP_KERNEL);
+        if (!dest) {
+                while (src_idx--)
+                        __free_page(xor_srcs[src_idx]);
+                return -ENOMEM;
+        }
+
+        /* Fill in src buffers */
+        for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
+                u8 *ptr = page_address(xor_srcs[src_idx]);
+                for (i = 0; i < PAGE_SIZE; i++)
+                        ptr[i] = (1 << src_idx);
+        }
+
+        for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
+                cmp_byte ^= (u8) (1 << src_idx);
+
+        cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
+                        (cmp_byte << 8) | cmp_byte;
+
+        memset(page_address(dest), 0, PAGE_SIZE);
+
+        dma_chan = container_of(dma->channels.next, struct dma_chan,
+                                device_node);
+        if (dma->device_alloc_chan_resources(dma_chan) < 1) {
+                err = -ENODEV;
+                goto out;
+        }
+
+        /* test xor */
+        dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
+        for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
+                dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
+                                           DMA_TO_DEVICE);
+        tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
+                                      IOAT_NUM_SRC_TEST, PAGE_SIZE,
+                                      DMA_PREP_INTERRUPT);
+
+        if (!tx) {
+                dev_err(dev, "Self-test xor prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test xor setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test xor timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
+        for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
+                u32 *ptr = page_address(dest);
+                if (ptr[i] != cmp_word) {
+                        dev_err(dev, "Self-test xor failed compare\n");
+                        err = -ENODEV;
+                        goto free_resources;
+                }
+        }
+        dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_TO_DEVICE);
+
+        /* skip validate if the capability is not present */
+        if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
+                goto free_resources;
+
+        /* validate the sources with the destintation page */
+        for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
+                xor_val_srcs[i] = xor_srcs[i];
+        xor_val_srcs[i] = dest;
+
+        xor_val_result = 1;
+
+        for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
+                dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
+                                           DMA_TO_DEVICE);
+        tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
+                                          IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                          &xor_val_result, DMA_PREP_INTERRUPT);
+        if (!tx) {
+                dev_err(dev, "Self-test zero prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test zero setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test validate timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (xor_val_result != 0) {
+                dev_err(dev, "Self-test validate failed compare\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        /* skip memset if the capability is not present */
+        if (!dma_has_cap(DMA_MEMSET, dma_chan->device->cap_mask))
+                goto free_resources;
+
+        /* test memset */
+        dma_addr = dma_map_page(dev, dest, 0,
+                        PAGE_SIZE, DMA_FROM_DEVICE);
+        tx = dma->device_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
+                                         DMA_PREP_INTERRUPT);
+        if (!tx) {
+                dev_err(dev, "Self-test memset prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test memset setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test memset timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
+                u32 *ptr = page_address(dest);
+                if (ptr[i]) {
+                        dev_err(dev, "Self-test memset failed compare\n");
+                        err = -ENODEV;
+                        goto free_resources;
+                }
+        }
+
+        /* test for non-zero parity sum */
+        xor_val_result = 0;
+        for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
+                dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
+                                           DMA_TO_DEVICE);
+        tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
+                                          IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                          &xor_val_result, DMA_PREP_INTERRUPT);
+        if (!tx) {
+                dev_err(dev, "Self-test 2nd zero prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test  2nd zero setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test 2nd validate timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (xor_val_result != SUM_CHECK_P_RESULT) {
+                dev_err(dev, "Self-test validate failed compare\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+free_resources:
+        dma->device_free_chan_resources(dma_chan);
+out:
+        src_idx = IOAT_NUM_SRC_TEST;
+        while (src_idx--)
+                __free_page(xor_srcs[src_idx]);
+        __free_page(dest);
+        return err;
+}
+
+static int __devinit ioat3_dma_self_test(struct ioatdma_device *device)
+{
+        int rc = ioat_dma_self_test(device);
+
+        if (rc)
+                return rc;
+
+        rc = ioat_xor_val_self_test(device);
+        if (rc)
+                return rc;
+
+        return 0;
+}
+
+int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
+{
+        struct pci_dev *pdev = device->pdev;
+        struct dma_device *dma;
+        struct dma_chan *c;
+        struct ioat_chan_common *chan;
+        bool is_raid_device = false;
+        int err;
+        u16 dev_id;
+        u32 cap;
+
+        device->enumerate_channels = ioat2_enumerate_channels;
+        device->self_test = ioat3_dma_self_test;
+        dma = &device->common;
+        dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
+        dma->device_issue_pending = ioat2_issue_pending;
+        dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
+        dma->device_free_chan_resources = ioat2_free_chan_resources;
+
+        dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
+        dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
+
+        cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET);
+        if (cap & IOAT_CAP_XOR) {
+                is_raid_device = true;
+                dma->max_xor = 8;
+                dma->xor_align = 2;
+
+                dma_cap_set(DMA_XOR, dma->cap_mask);
+                dma->device_prep_dma_xor = ioat3_prep_xor;
+
+                dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
+                dma->device_prep_dma_xor_val = ioat3_prep_xor_val;
+        }
+        if (cap & IOAT_CAP_PQ) {
+                is_raid_device = true;
+                dma_set_maxpq(dma, 8, 0);
+                dma->pq_align = 2;
+
+                dma_cap_set(DMA_PQ, dma->cap_mask);
+                dma->device_prep_dma_pq = ioat3_prep_pq;
+
+                dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
+                dma->device_prep_dma_pq_val = ioat3_prep_pq_val;
+
+                if (!(cap & IOAT_CAP_XOR)) {
+                        dma->max_xor = 8;
+                        dma->xor_align = 2;
+
+                        dma_cap_set(DMA_XOR, dma->cap_mask);
+                        dma->device_prep_dma_xor = ioat3_prep_pqxor;
+
+                        dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
+                        dma->device_prep_dma_xor_val = ioat3_prep_pqxor_val;
+                }
+        }
+        if (is_raid_device && (cap & IOAT_CAP_FILL_BLOCK)) {
+                dma_cap_set(DMA_MEMSET, dma->cap_mask);
+                dma->device_prep_dma_memset = ioat3_prep_memset_lock;
+        }
+
+
+        if (is_raid_device) {
+                dma->device_is_tx_complete = ioat3_is_complete;
+                device->cleanup_tasklet = ioat3_cleanup_tasklet;
+                device->timer_fn = ioat3_timer_event;
+        } else {
+                dma->device_is_tx_complete = ioat2_is_complete;
+                device->cleanup_tasklet = ioat2_cleanup_tasklet;
+                device->timer_fn = ioat2_timer_event;
+        }
+
+        /* -= IOAT ver.3 workarounds =- */
+        /* Write CHANERRMSK_INT with 3E07h to mask out the errors
+         * that can cause stability issues for IOAT ver.3
+         */
+        pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
+
+        /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
+         * (workaround for spurious config parity error after restart)
+         */
+        pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
+        if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
+                pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
+
+        err = ioat_probe(device);
+        if (err)
+                return err;
+        ioat_set_tcp_copy_break(262144);
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                chan = to_chan_common(c);
+                writel(IOAT_DMA_DCA_ANY_CPU,
+                       chan->reg_base + IOAT_DCACTRL_OFFSET);
+        }
+
+        err = ioat_register(device);
+        if (err)
+                return err;
+
+        ioat_kobject_add(device, &ioat2_ktype);
+
+        if (dca)
+                device->dca = ioat3_dca_init(pdev, device->reg_base);
+
+        return 0;
+}
diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h
new file mode 100644
index 000000000000..99afb12bd409
--- /dev/null
+++ b/drivers/dma/ioat/hw.h
@@ -0,0 +1,215 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef _IOAT_HW_H_
+#define _IOAT_HW_H_
+
+/* PCI Configuration Space Values */
+#define IOAT_PCI_VID            0x8086
+#define IOAT_MMIO_BAR           0
+
+/* CB device ID's */
+#define IOAT_PCI_DID_5000       0x1A38
+#define IOAT_PCI_DID_CNB        0x360B
+#define IOAT_PCI_DID_SCNB       0x65FF
+#define IOAT_PCI_DID_SNB        0x402F
+
+#define IOAT_PCI_RID            0x00
+#define IOAT_PCI_SVID           0x8086
+#define IOAT_PCI_SID            0x8086
+#define IOAT_VER_1_2            0x12    /* Version 1.2 */
+#define IOAT_VER_2_0            0x20    /* Version 2.0 */
+#define IOAT_VER_3_0            0x30    /* Version 3.0 */
+#define IOAT_VER_3_2            0x32    /* Version 3.2 */
+
+struct ioat_dma_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int null:1;
+                        unsigned int src_brk:1;
+                        unsigned int dest_brk:1;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int rsvd2:13;
+                        #define IOAT_OP_COPY 0x00
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        dst_addr;
+        uint64_t        next;
+        uint64_t        rsv1;
+        uint64_t        rsv2;
+        /* store some driver data in an unused portion of the descriptor */
+        union {
+                uint64_t        user1;
+                uint64_t        tx_cnt;
+        };
+        uint64_t        user2;
+};
+
+struct ioat_fill_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int rsvd:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int rsvd2:2;
+                        unsigned int dest_brk:1;
+                        unsigned int bundle:1;
+                        unsigned int rsvd4:15;
+                        #define IOAT_OP_FILL 0x01
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_data;
+        uint64_t        dst_addr;
+        uint64_t        next;
+        uint64_t        rsv1;
+        uint64_t        next_dst_addr;
+        uint64_t        user1;
+        uint64_t        user2;
+};
+
+struct ioat_xor_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int src_cnt:3;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int rsvd:13;
+                        #define IOAT_OP_XOR 0x87
+                        #define IOAT_OP_XOR_VAL 0x88
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        dst_addr;
+        uint64_t        next;
+        uint64_t        src_addr2;
+        uint64_t        src_addr3;
+        uint64_t        src_addr4;
+        uint64_t        src_addr5;
+};
+
+struct ioat_xor_ext_descriptor {
+        uint64_t        src_addr6;
+        uint64_t        src_addr7;
+        uint64_t        src_addr8;
+        uint64_t        next;
+        uint64_t        rsvd[4];
+};
+
+struct ioat_pq_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int src_cnt:3;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int p_disable:1;
+                        unsigned int q_disable:1;
+                        unsigned int rsvd:11;
+                        #define IOAT_OP_PQ 0x89
+                        #define IOAT_OP_PQ_VAL 0x8a
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        p_addr;
+        uint64_t        next;
+        uint64_t        src_addr2;
+        uint64_t        src_addr3;
+        uint8_t         coef[8];
+        uint64_t        q_addr;
+};
+
+struct ioat_pq_ext_descriptor {
+        uint64_t        src_addr4;
+        uint64_t        src_addr5;
+        uint64_t        src_addr6;
+        uint64_t        next;
+        uint64_t        src_addr7;
+        uint64_t        src_addr8;
+        uint64_t        rsvd[2];
+};
+
+struct ioat_pq_update_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int src_cnt:3;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int p_disable:1;
+                        unsigned int q_disable:1;
+                        unsigned int rsvd:3;
+                        unsigned int coef:8;
+                        #define IOAT_OP_PQ_UP 0x8b
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        p_addr;
+        uint64_t        next;
+        uint64_t        src_addr2;
+        uint64_t        p_src;
+        uint64_t        q_src;
+        uint64_t        q_addr;
+};
+
+struct ioat_raw_descriptor {
+        uint64_t        field[8];
+};
+#endif
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
new file mode 100644
index 000000000000..d545fae30f37
--- /dev/null
+++ b/drivers/dma/ioat/pci.c
@@ -0,0 +1,210 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2007 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine, which does asynchronous
+ * copy operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dca.h>
+#include "dma.h"
+#include "dma_v2.h"
+#include "registers.h"
+#include "hw.h"
+
+MODULE_VERSION(IOAT_DMA_VERSION);
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Intel Corporation");
+
+static struct pci_device_id ioat_pci_tbl[] = {
+        /* I/OAT v1 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB)  },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
+        { PCI_VDEVICE(UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
+
+        /* I/OAT v2 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
+
+        /* I/OAT v3 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
+
+        /* I/OAT v3.2 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
+
+        { 0, }
+};
+MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
+
+static int __devinit ioat_pci_probe(struct pci_dev *pdev,
+                                    const struct pci_device_id *id);
+static void __devexit ioat_remove(struct pci_dev *pdev);
+
+static int ioat_dca_enabled = 1;
+module_param(ioat_dca_enabled, int, 0644);
+MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
+
+struct kmem_cache *ioat2_cache;
+
+#define DRV_NAME "ioatdma"
+
+static struct pci_driver ioat_pci_driver = {
+        .name           = DRV_NAME,
+        .id_table       = ioat_pci_tbl,
+        .probe          = ioat_pci_probe,
+        .remove         = __devexit_p(ioat_remove),
+};
+
+static struct ioatdma_device *
+alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
+{
+        struct device *dev = &pdev->dev;
+        struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
+
+        if (!d)
+                return NULL;
+        d->pdev = pdev;
+        d->reg_base = iobase;
+        return d;
+}
+
+static int __devinit ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+        void __iomem * const *iomap;
+        struct device *dev = &pdev->dev;
+        struct ioatdma_device *device;
+        int err;
+
+        err = pcim_enable_device(pdev);
+        if (err)
+                return err;
+
+        err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
+        if (err)
+                return err;
+        iomap = pcim_iomap_table(pdev);
+        if (!iomap)
+                return -ENOMEM;
+
+        err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+        if (err)
+                err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+        if (err)
+                return err;
+
+        err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+        if (err)
+                err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+        if (err)
+                return err;
+
+        device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL);
+        if (!device)
+                return -ENOMEM;
+
+        pci_set_master(pdev);
+
+        device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
+        if (!device)
+                return -ENOMEM;
+        pci_set_drvdata(pdev, device);
+
+        device->version = readb(device->reg_base + IOAT_VER_OFFSET);
+        if (device->version == IOAT_VER_1_2)
+                err = ioat1_dma_probe(device, ioat_dca_enabled);
+        else if (device->version == IOAT_VER_2_0)
+                err = ioat2_dma_probe(device, ioat_dca_enabled);
+        else if (device->version >= IOAT_VER_3_0)
+                err = ioat3_dma_probe(device, ioat_dca_enabled);
+        else
+                return -ENODEV;
+
+        if (err) {
+                dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
+                return -ENODEV;
+        }
+
+        return 0;
+}
+
+static void __devexit ioat_remove(struct pci_dev *pdev)
+{
+        struct ioatdma_device *device = pci_get_drvdata(pdev);
+
+        if (!device)
+                return;
+
+        dev_err(&pdev->dev, "Removing dma and dca services\n");
+        if (device->dca) {
+                unregister_dca_provider(device->dca, &pdev->dev);
+                free_dca_provider(device->dca);
+                device->dca = NULL;
+        }
+        ioat_dma_remove(device);
+}
+
+static int __init ioat_init_module(void)
+{
+        int err;
+
+        pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
+                DRV_NAME, IOAT_DMA_VERSION);
+
+        ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent),
+                                        0, SLAB_HWCACHE_ALIGN, NULL);
+        if (!ioat2_cache)
+                return -ENOMEM;
+
+        err = pci_register_driver(&ioat_pci_driver);
+        if (err)
+                kmem_cache_destroy(ioat2_cache);
+
+        return err;
+}
+module_init(ioat_init_module);
+
+static void __exit ioat_exit_module(void)
+{
+        pci_unregister_driver(&ioat_pci_driver);
+        kmem_cache_destroy(ioat2_cache);
+}
+module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioatdma_registers.h b/drivers/dma/ioat/registers.h
index 49bc277424f8..63038e18ab03 100644
--- a/drivers/dma/ioatdma_registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -64,18 +64,37 @@
 
 #define IOAT_DEVICE_STATUS_OFFSET               0x0E    /* 16-bit */
 #define IOAT_DEVICE_STATUS_DEGRADED_MODE        0x0001
+#define IOAT_DEVICE_MMIO_RESTRICTED             0x0002
+#define IOAT_DEVICE_MEMORY_BYPASS               0x0004
+#define IOAT_DEVICE_ADDRESS_REMAPPING           0x0008
+
+#define IOAT_DMA_CAP_OFFSET                     0x10    /* 32-bit */
+#define IOAT_CAP_PAGE_BREAK                     0x00000001
+#define IOAT_CAP_CRC                            0x00000002
+#define IOAT_CAP_SKIP_MARKER                    0x00000004
+#define IOAT_CAP_DCA                            0x00000010
+#define IOAT_CAP_CRC_MOVE                       0x00000020
+#define IOAT_CAP_FILL_BLOCK                     0x00000040
+#define IOAT_CAP_APIC                           0x00000080
+#define IOAT_CAP_XOR                            0x00000100
+#define IOAT_CAP_PQ                             0x00000200
 
 #define IOAT_CHANNEL_MMIO_SIZE                  0x80    /* Each Channel MMIO space is this size */
 
 /* DMA Channel Registers */
 #define IOAT_CHANCTRL_OFFSET                    0x00    /* 16-bit Channel Control Register */
 #define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK     0xF000
+#define IOAT3_CHANCTRL_COMPL_DCA_EN             0x0200
 #define IOAT_CHANCTRL_CHANNEL_IN_USE            0x0100
 #define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL     0x0020
 #define IOAT_CHANCTRL_ERR_INT_EN                0x0010
 #define IOAT_CHANCTRL_ANY_ERR_ABORT_EN          0x0008
 #define IOAT_CHANCTRL_ERR_COMPLETION_EN         0x0004
-#define IOAT_CHANCTRL_INT_DISABLE               0x0001
+#define IOAT_CHANCTRL_INT_REARM                 0x0001
+#define IOAT_CHANCTRL_RUN                       (IOAT_CHANCTRL_INT_REARM |\
+                                                 IOAT_CHANCTRL_ERR_COMPLETION_EN |\
+                                                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\
+                                                 IOAT_CHANCTRL_ERR_INT_EN)
 
 #define IOAT_DMA_COMP_OFFSET                    0x02    /* 16-bit DMA channel compatibility */
 #define IOAT_DMA_COMP_V1                        0x0001  /* Compatibility with DMA version 1 */
@@ -94,14 +113,14 @@
 #define IOAT2_CHANSTS_OFFSET_HIGH       0x0C
 #define IOAT_CHANSTS_OFFSET_HIGH(ver)           ((ver) < IOAT_VER_2_0 \
                                                 ? IOAT1_CHANSTS_OFFSET_HIGH : IOAT2_CHANSTS_OFFSET_HIGH)
-#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR  ~0x3F
-#define IOAT_CHANSTS_SOFT_ERR                   0x0000000000000010
-#define IOAT_CHANSTS_UNAFFILIATED_ERR           0x0000000000000008
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS        0x0000000000000007
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE   0x1
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED      0x2
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
+#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR  (~0x3fULL)
+#define IOAT_CHANSTS_SOFT_ERR                   0x10ULL
+#define IOAT_CHANSTS_UNAFFILIATED_ERR           0x8ULL
+#define IOAT_CHANSTS_STATUS     0x7ULL
+#define IOAT_CHANSTS_ACTIVE     0x0
+#define IOAT_CHANSTS_DONE       0x1
+#define IOAT_CHANSTS_SUSPENDED  0x2
+#define IOAT_CHANSTS_HALTED     0x3
 
 
 
@@ -204,22 +223,27 @@
 #define IOAT_CDAR_OFFSET_HIGH                   0x24
 
 #define IOAT_CHANERR_OFFSET                     0x28    /* 32-bit Channel Error Register */
-#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR  0x0001
-#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR   0x0004
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR      0x0008
+#define IOAT_CHANERR_SRC_ADDR_ERR       0x0001
+#define IOAT_CHANERR_DEST_ADDR_ERR      0x0002
+#define IOAT_CHANERR_NEXT_ADDR_ERR      0x0004
+#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR        0x0008
 #define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR       0x0010
 #define IOAT_CHANERR_CHANCMD_ERR                0x0020
 #define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR   0x0040
 #define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR       0x0080
 #define IOAT_CHANERR_READ_DATA_ERR              0x0100
 #define IOAT_CHANERR_WRITE_DATA_ERR             0x0200
-#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR     0x0400
-#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR      0x0800
+#define IOAT_CHANERR_CONTROL_ERR        0x0400
+#define IOAT_CHANERR_LENGTH_ERR 0x0800
 #define IOAT_CHANERR_COMPLETION_ADDR_ERR        0x1000
 #define IOAT_CHANERR_INT_CONFIGURATION_ERR      0x2000
 #define IOAT_CHANERR_SOFT_ERR                   0x4000
 #define IOAT_CHANERR_UNAFFILIATED_ERR           0x8000
+#define IOAT_CHANERR_XOR_P_OR_CRC_ERR           0x10000
+#define IOAT_CHANERR_XOR_Q_ERR                  0x20000
+#define IOAT_CHANERR_DESCRIPTOR_COUNT_ERR       0x40000
+
+#define IOAT_CHANERR_HANDLE_MASK (IOAT_CHANERR_XOR_P_OR_CRC_ERR | IOAT_CHANERR_XOR_Q_ERR)
 
 #define IOAT_CHANERR_MASK_OFFSET                0x2C    /* 32-bit Channel Error Register */
 
diff --git a/drivers/dma/ioat_dma.c b/drivers/dma/ioat_dma.c
deleted file mode 100644
index a600fc0f7962..000000000000
--- a/drivers/dma/ioat_dma.c
+++ /dev/null
@@ -1,1741 +0,0 @@
-/*
- * Intel I/OAT DMA Linux driver
- * Copyright(c) 2004 - 2009 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- */
-
-/*
- * This driver supports an Intel I/OAT DMA engine, which does asynchronous
- * copy operations.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/dmaengine.h>
-#include <linux/delay.h>
-#include <linux/dma-mapping.h>
-#include <linux/workqueue.h>
-#include <linux/i7300_idle.h>
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
-#include "ioatdma_hw.h"
-
-#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
-#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
-#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
-#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
-
-#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
-static int ioat_pending_level = 4;
-module_param(ioat_pending_level, int, 0644);
-MODULE_PARM_DESC(ioat_pending_level,
-                 "high-water mark for pushing ioat descriptors (default: 4)");
-
-#define RESET_DELAY  msecs_to_jiffies(100)
-#define WATCHDOG_DELAY  round_jiffies(msecs_to_jiffies(2000))
-static void ioat_dma_chan_reset_part2(struct work_struct *work);
-static void ioat_dma_chan_watchdog(struct work_struct *work);
-
-/*
- * workaround for IOAT ver.3.0 null descriptor issue
- * (channel returns error when size is 0)
- */
-#define NULL_DESC_BUFFER_SIZE 1
-
-/* internal functions */
-static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
-static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
-
-static struct ioat_desc_sw *
-ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
-static struct ioat_desc_sw *
-ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
-
-static inline struct ioat_dma_chan *ioat_lookup_chan_by_index(
-                                                struct ioatdma_device *device,
-                                                int index)
-{
-        return device->idx[index];
-}
-
-/**
- * ioat_dma_do_interrupt - handler used for single vector interrupt mode
- * @irq: interrupt id
- * @data: interrupt data
- */
-static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
-{
-        struct ioatdma_device *instance = data;
-        struct ioat_dma_chan *ioat_chan;
-        unsigned long attnstatus;
-        int bit;
-        u8 intrctrl;
-
-        intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
-
-        if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
-                return IRQ_NONE;
-
-        if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
-                writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
-                return IRQ_NONE;
-        }
-
-        attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
-        for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
-                ioat_chan = ioat_lookup_chan_by_index(instance, bit);
-                tasklet_schedule(&ioat_chan->cleanup_task);
-        }
-
-        writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
-        return IRQ_HANDLED;
-}
-
-/**
- * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
- * @irq: interrupt id
- * @data: interrupt data
- */
-static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
-{
-        struct ioat_dma_chan *ioat_chan = data;
-
-        tasklet_schedule(&ioat_chan->cleanup_task);
-
-        return IRQ_HANDLED;
-}
-
-static void ioat_dma_cleanup_tasklet(unsigned long data);
-
-/**
- * ioat_dma_enumerate_channels - find and initialize the device's channels
- * @device: the device to be enumerated
- */
-static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
-{
-        u8 xfercap_scale;
-        u32 xfercap;
-        int i;
-        struct ioat_dma_chan *ioat_chan;
-
-        /*
-         * IOAT ver.3 workarounds
-         */
-        if (device->version == IOAT_VER_3_0) {
-                u32 chan_err_mask;
-                u16 dev_id;
-                u32 dmauncerrsts;
-
-                /*
-                 * Write CHANERRMSK_INT with 3E07h to mask out the errors
-                 * that can cause stability issues for IOAT ver.3
-                 */
-                chan_err_mask = 0x3E07;
-                pci_write_config_dword(device->pdev,
-                        IOAT_PCI_CHANERRMASK_INT_OFFSET,
-                        chan_err_mask);
-
-                /*
-                 * Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
-                 * (workaround for spurious config parity error after restart)
-                 */
-                pci_read_config_word(device->pdev,
-                        IOAT_PCI_DEVICE_ID_OFFSET,
-                        &dev_id);
-                if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
-                        dmauncerrsts = 0x10;
-                        pci_write_config_dword(device->pdev,
-                                IOAT_PCI_DMAUNCERRSTS_OFFSET,
-                                dmauncerrsts);
-                }
-        }
-
-        device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
-        xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
-        xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
-
-#ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
-        if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) {
-                device->common.chancnt--;
-        }
-#endif
-        for (i = 0; i < device->common.chancnt; i++) {
-                ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
-                if (!ioat_chan) {
-                        device->common.chancnt = i;
-                        break;
-                }
-
-                ioat_chan->device = device;
-                ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
-                ioat_chan->xfercap = xfercap;
-                ioat_chan->desccount = 0;
-                INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
-                if (ioat_chan->device->version == IOAT_VER_2_0)
-                        writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE |
-                               IOAT_DMA_DCA_ANY_CPU,
-                               ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
-                else if (ioat_chan->device->version == IOAT_VER_3_0)
-                        writel(IOAT_DMA_DCA_ANY_CPU,
-                               ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
-                spin_lock_init(&ioat_chan->cleanup_lock);
-                spin_lock_init(&ioat_chan->desc_lock);
-                INIT_LIST_HEAD(&ioat_chan->free_desc);
-                INIT_LIST_HEAD(&ioat_chan->used_desc);
-                /* This should be made common somewhere in dmaengine.c */
-                ioat_chan->common.device = &device->common;
-                list_add_tail(&ioat_chan->common.device_node,
-                              &device->common.channels);
-                device->idx[i] = ioat_chan;
-                tasklet_init(&ioat_chan->cleanup_task,
-                             ioat_dma_cleanup_tasklet,
-                             (unsigned long) ioat_chan);
-                tasklet_disable(&ioat_chan->cleanup_task);
-        }
-        return device->common.chancnt;
-}
-
-/**
- * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
- *                                 descriptors to hw
- * @chan: DMA channel handle
- */
-static inline void __ioat1_dma_memcpy_issue_pending(
-                                                struct ioat_dma_chan *ioat_chan)
-{
-        ioat_chan->pending = 0;
-        writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);
-}
-
-static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-        if (ioat_chan->pending > 0) {
-                spin_lock_bh(&ioat_chan->desc_lock);
-                __ioat1_dma_memcpy_issue_pending(ioat_chan);
-                spin_unlock_bh(&ioat_chan->desc_lock);
-        }
-}
-
-static inline void __ioat2_dma_memcpy_issue_pending(
-                                                struct ioat_dma_chan *ioat_chan)
-{
-        ioat_chan->pending = 0;
-        writew(ioat_chan->dmacount,
-               ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
-}
-
-static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-        if (ioat_chan->pending > 0) {
-                spin_lock_bh(&ioat_chan->desc_lock);
-                __ioat2_dma_memcpy_issue_pending(ioat_chan);
-                spin_unlock_bh(&ioat_chan->desc_lock);
-        }
-}
-
-
-/**
- * ioat_dma_chan_reset_part2 - reinit the channel after a reset
- */
-static void ioat_dma_chan_reset_part2(struct work_struct *work)
-{
-        struct ioat_dma_chan *ioat_chan =
-                container_of(work, struct ioat_dma_chan, work.work);
-        struct ioat_desc_sw *desc;
-
-        spin_lock_bh(&ioat_chan->cleanup_lock);
-        spin_lock_bh(&ioat_chan->desc_lock);
-
-        ioat_chan->completion_virt->low = 0;
-        ioat_chan->completion_virt->high = 0;
-        ioat_chan->pending = 0;
-
-        /*
-         * count the descriptors waiting, and be sure to do it
-         * right for both the CB1 line and the CB2 ring
-         */
-        ioat_chan->dmacount = 0;
-        if (ioat_chan->used_desc.prev) {
-                desc = to_ioat_desc(ioat_chan->used_desc.prev);
-                do {
-                        ioat_chan->dmacount++;
-                        desc = to_ioat_desc(desc->node.next);
-                } while (&desc->node != ioat_chan->used_desc.next);
-        }
-
-        /*
-         * write the new starting descriptor address
-         * this puts channel engine into ARMED state
-         */
-        desc = to_ioat_desc(ioat_chan->used_desc.prev);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
-                writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
-                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-                break;
-        case IOAT_VER_2_0:
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
-
-                /* tell the engine to go with what's left to be done */
-                writew(ioat_chan->dmacount,
-                       ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
-
-                break;
-        }
-        dev_err(&ioat_chan->device->pdev->dev,
-                "chan%d reset - %d descs waiting, %d total desc\n",
-                chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
-
-        spin_unlock_bh(&ioat_chan->desc_lock);
-        spin_unlock_bh(&ioat_chan->cleanup_lock);
-}
-
-/**
- * ioat_dma_reset_channel - restart a channel
- * @ioat_chan: IOAT DMA channel handle
- */
-static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat_chan)
-{
-        u32 chansts, chanerr;
-
-        if (!ioat_chan->used_desc.prev)
-                return;
-
-        chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        chansts = (ioat_chan->completion_virt->low
-                                        & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
-        if (chanerr) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
-                        chan_num(ioat_chan), chansts, chanerr);
-                writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        }
-
-        /*
-         * whack it upside the head with a reset
-         * and wait for things to settle out.
-         * force the pending count to a really big negative
-         * to make sure no one forces an issue_pending
-         * while we're waiting.
-         */
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        ioat_chan->pending = INT_MIN;
-        writeb(IOAT_CHANCMD_RESET,
-               ioat_chan->reg_base
-               + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        /* schedule the 2nd half instead of sleeping a long time */
-        schedule_delayed_work(&ioat_chan->work, RESET_DELAY);
-}
-
-/**
- * ioat_dma_chan_watchdog - watch for stuck channels
- */
-static void ioat_dma_chan_watchdog(struct work_struct *work)
-{
-        struct ioatdma_device *device =
-                container_of(work, struct ioatdma_device, work.work);
-        struct ioat_dma_chan *ioat_chan;
-        int i;
-
-        union {
-                u64 full;
-                struct {
-                        u32 low;
-                        u32 high;
-                };
-        } completion_hw;
-        unsigned long compl_desc_addr_hw;
-
-        for (i = 0; i < device->common.chancnt; i++) {
-                ioat_chan = ioat_lookup_chan_by_index(device, i);
-
-                if (ioat_chan->device->version == IOAT_VER_1_2
-                        /* have we started processing anything yet */
-                    && ioat_chan->last_completion
-                        /* have we completed any since last watchdog cycle? */
-                    && (ioat_chan->last_completion ==
-                                ioat_chan->watchdog_completion)
-                        /* has TCP stuck on one cookie since last watchdog? */
-                    && (ioat_chan->watchdog_tcp_cookie ==
-                                ioat_chan->watchdog_last_tcp_cookie)
-                    && (ioat_chan->watchdog_tcp_cookie !=
-                                ioat_chan->completed_cookie)
-                        /* is there something in the chain to be processed? */
-                        /* CB1 chain always has at least the last one processed */
-                    && (ioat_chan->used_desc.prev != ioat_chan->used_desc.next)
-                    && ioat_chan->pending == 0) {
-
-                        /*
-                         * check CHANSTS register for completed
-                         * descriptor address.
-                         * if it is different than completion writeback,
-                         * it is not zero
-                         * and it has changed since the last watchdog
-                         *     we can assume that channel
-                         *     is still working correctly
-                         *     and the problem is in completion writeback.
-                         *     update completion writeback
-                         *     with actual CHANSTS value
-                         * else
-                         *     try resetting the channel
-                         */
-
-                        completion_hw.low = readl(ioat_chan->reg_base +
-                                IOAT_CHANSTS_OFFSET_LOW(ioat_chan->device->version));
-                        completion_hw.high = readl(ioat_chan->reg_base +
-                                IOAT_CHANSTS_OFFSET_HIGH(ioat_chan->device->version));
-#if (BITS_PER_LONG == 64)
-                        compl_desc_addr_hw =
-                                completion_hw.full
-                                & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
-                        compl_desc_addr_hw =
-                                completion_hw.low & IOAT_LOW_COMPLETION_MASK;
-#endif
-
-                        if ((compl_desc_addr_hw != 0)
-                           && (compl_desc_addr_hw != ioat_chan->watchdog_completion)
-                           && (compl_desc_addr_hw != ioat_chan->last_compl_desc_addr_hw)) {
-                                ioat_chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
-                                ioat_chan->completion_virt->low = completion_hw.low;
-                                ioat_chan->completion_virt->high = completion_hw.high;
-                        } else {
-                                ioat_dma_reset_channel(ioat_chan);
-                                ioat_chan->watchdog_completion = 0;
-                                ioat_chan->last_compl_desc_addr_hw = 0;
-                        }
-
-                /*
-                 * for version 2.0 if there are descriptors yet to be processed
-                 * and the last completed hasn't changed since the last watchdog
-                 *      if they haven't hit the pending level
-                 *          issue the pending to push them through
-                 *      else
-                 *          try resetting the channel
-                 */
-                } else if (ioat_chan->device->version == IOAT_VER_2_0
-                    && ioat_chan->used_desc.prev
-                    && ioat_chan->last_completion
-                    && ioat_chan->last_completion == ioat_chan->watchdog_completion) {
-
-                        if (ioat_chan->pending < ioat_pending_level)
-                                ioat2_dma_memcpy_issue_pending(&ioat_chan->common);
-                        else {
-                                ioat_dma_reset_channel(ioat_chan);
-                                ioat_chan->watchdog_completion = 0;
-                        }
-                } else {
-                        ioat_chan->last_compl_desc_addr_hw = 0;
-                        ioat_chan->watchdog_completion
-                                        = ioat_chan->last_completion;
-                }
-
-                ioat_chan->watchdog_last_tcp_cookie =
-                        ioat_chan->watchdog_tcp_cookie;
-        }
-
-        schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-}
-
-static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
-        struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
-        struct ioat_desc_sw *prev, *new;
-        struct ioat_dma_descriptor *hw;
-        dma_cookie_t cookie;
-        LIST_HEAD(new_chain);
-        u32 copy;
-        size_t len;
-        dma_addr_t src, dst;
-        unsigned long orig_flags;
-        unsigned int desc_count = 0;
-
-        /* src and dest and len are stored in the initial descriptor */
-        len = first->len;
-        src = first->src;
-        dst = first->dst;
-        orig_flags = first->async_tx.flags;
-        new = first;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        prev = to_ioat_desc(ioat_chan->used_desc.prev);
-        prefetch(prev->hw);
-        do {
-                copy = min_t(size_t, len, ioat_chan->xfercap);
-
-                async_tx_ack(&new->async_tx);
-
-                hw = new->hw;
-                hw->size = copy;
-                hw->ctl = 0;
-                hw->src_addr = src;
-                hw->dst_addr = dst;
-                hw->next = 0;
-
-                /* chain together the physical address list for the HW */
-                wmb();
-                prev->hw->next = (u64) new->async_tx.phys;
-
-                len -= copy;
-                dst += copy;
-                src += copy;
-
-                list_add_tail(&new->node, &new_chain);
-                desc_count++;
-                prev = new;
-        } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
-
-        if (!new) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "tx submit failed\n");
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                return -ENOMEM;
-        }
-
-        hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
-        if (first->async_tx.callback) {
-                hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
-                if (first != new) {
-                        /* move callback into to last desc */
-                        new->async_tx.callback = first->async_tx.callback;
-                        new->async_tx.callback_param
-                                        = first->async_tx.callback_param;
-                        first->async_tx.callback = NULL;
-                        first->async_tx.callback_param = NULL;
-                }
-        }
-
-        new->tx_cnt = desc_count;
-        new->async_tx.flags = orig_flags; /* client is in control of this ack */
-
-        /* store the original values for use in later cleanup */
-        if (new != first) {
-                new->src = first->src;
-                new->dst = first->dst;
-                new->len = first->len;
-        }
-
-        /* cookie incr and addition to used_list must be atomic */
-        cookie = ioat_chan->common.cookie;
-        cookie++;
-        if (cookie < 0)
-                cookie = 1;
-        ioat_chan->common.cookie = new->async_tx.cookie = cookie;
-
-        /* write address into NextDescriptor field of last desc in chain */
-        to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
-                                                        first->async_tx.phys;
-        list_splice_tail(&new_chain, &ioat_chan->used_desc);
-
-        ioat_chan->dmacount += desc_count;
-        ioat_chan->pending += desc_count;
-        if (ioat_chan->pending >= ioat_pending_level)
-                __ioat1_dma_memcpy_issue_pending(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        return cookie;
-}
-
-static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
-        struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
-        struct ioat_desc_sw *new;
-        struct ioat_dma_descriptor *hw;
-        dma_cookie_t cookie;
-        u32 copy;
-        size_t len;
-        dma_addr_t src, dst;
-        unsigned long orig_flags;
-        unsigned int desc_count = 0;
-
-        /* src and dest and len are stored in the initial descriptor */
-        len = first->len;
-        src = first->src;
-        dst = first->dst;
-        orig_flags = first->async_tx.flags;
-        new = first;
-
-        /*
-         * ioat_chan->desc_lock is still in force in version 2 path
-         * it gets unlocked at end of this function
-         */
-        do {
-                copy = min_t(size_t, len, ioat_chan->xfercap);
-
-                async_tx_ack(&new->async_tx);
-
-                hw = new->hw;
-                hw->size = copy;
-                hw->ctl = 0;
-                hw->src_addr = src;
-                hw->dst_addr = dst;
-
-                len -= copy;
-                dst += copy;
-                src += copy;
-                desc_count++;
-        } while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));
-
-        if (!new) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "tx submit failed\n");
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                return -ENOMEM;
-        }
-
-        hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
-        if (first->async_tx.callback) {
-                hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
-                if (first != new) {
-                        /* move callback into to last desc */
-                        new->async_tx.callback = first->async_tx.callback;
-                        new->async_tx.callback_param
-                                        = first->async_tx.callback_param;
-                        first->async_tx.callback = NULL;
-                        first->async_tx.callback_param = NULL;
-                }
-        }
-
-        new->tx_cnt = desc_count;
-        new->async_tx.flags = orig_flags; /* client is in control of this ack */
-
-        /* store the original values for use in later cleanup */
-        if (new != first) {
-                new->src = first->src;
-                new->dst = first->dst;
-                new->len = first->len;
-        }
-
-        /* cookie incr and addition to used_list must be atomic */
-        cookie = ioat_chan->common.cookie;
-        cookie++;
-        if (cookie < 0)
-                cookie = 1;
-        ioat_chan->common.cookie = new->async_tx.cookie = cookie;
-
-        ioat_chan->dmacount += desc_count;
-        ioat_chan->pending += desc_count;
-        if (ioat_chan->pending >= ioat_pending_level)
-                __ioat2_dma_memcpy_issue_pending(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        return cookie;
-}
-
-/**
- * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
- * @ioat_chan: the channel supplying the memory pool for the descriptors
- * @flags: allocation flags
- */
-static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
-                                        struct ioat_dma_chan *ioat_chan,
-                                        gfp_t flags)
-{
-        struct ioat_dma_descriptor *desc;
-        struct ioat_desc_sw *desc_sw;
-        struct ioatdma_device *ioatdma_device;
-        dma_addr_t phys;
-
-        ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
-        desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
-        if (unlikely(!desc))
-                return NULL;
-
-        desc_sw = kzalloc(sizeof(*desc_sw), flags);
-        if (unlikely(!desc_sw)) {
-                pci_pool_free(ioatdma_device->dma_pool, desc, phys);
-                return NULL;
-        }
-
-        memset(desc, 0, sizeof(*desc));
-        dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                desc_sw->async_tx.tx_submit = ioat1_tx_submit;
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                desc_sw->async_tx.tx_submit = ioat2_tx_submit;
-                break;
-        }
-
-        desc_sw->hw = desc;
-        desc_sw->async_tx.phys = phys;
-
-        return desc_sw;
-}
-
-static int ioat_initial_desc_count = 256;
-module_param(ioat_initial_desc_count, int, 0644);
-MODULE_PARM_DESC(ioat_initial_desc_count,
-                 "initial descriptors per channel (default: 256)");
-
-/**
- * ioat2_dma_massage_chan_desc - link the descriptors into a circle
- * @ioat_chan: the channel to be massaged
- */
-static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *desc, *_desc;
-
-        /* setup used_desc */
-        ioat_chan->used_desc.next = ioat_chan->free_desc.next;
-        ioat_chan->used_desc.prev = NULL;
-
-        /* pull free_desc out of the circle so that every node is a hw
-         * descriptor, but leave it pointing to the list
-         */
-        ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;
-        ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;
-
-        /* circle link the hw descriptors */
-        desc = to_ioat_desc(ioat_chan->free_desc.next);
-        desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;
-        list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {
-                desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;
-        }
-}
-
-/**
- * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
- * @chan: the channel to be filled out
- */
-static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioat_desc_sw *desc;
-        u16 chanctrl;
-        u32 chanerr;
-        int i;
-        LIST_HEAD(tmp_list);
-
-        /* have we already been set up? */
-        if (!list_empty(&ioat_chan->free_desc))
-                return ioat_chan->desccount;
-
-        /* Setup register to interrupt and write completion status on error */
-        chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
-                IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
-                IOAT_CHANCTRL_ERR_COMPLETION_EN;
-        writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
-
-        chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        if (chanerr) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "CHANERR = %x, clearing\n", chanerr);
-                writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        }
-
-        /* Allocate descriptors */
-        for (i = 0; i < ioat_initial_desc_count; i++) {
-                desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
-                if (!desc) {
-                        dev_err(&ioat_chan->device->pdev->dev,
-                                "Only %d initial descriptors\n", i);
-                        break;
-                }
-                list_add_tail(&desc->node, &tmp_list);
-        }
-        spin_lock_bh(&ioat_chan->desc_lock);
-        ioat_chan->desccount = i;
-        list_splice(&tmp_list, &ioat_chan->free_desc);
-        if (ioat_chan->device->version != IOAT_VER_1_2)
-                ioat2_dma_massage_chan_desc(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        /* allocate a completion writeback area */
-        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
-        ioat_chan->completion_virt =
-                pci_pool_alloc(ioat_chan->device->completion_pool,
-                               GFP_KERNEL,
-                               &ioat_chan->completion_addr);
-        memset(ioat_chan->completion_virt, 0,
-               sizeof(*ioat_chan->completion_virt));
-        writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
-               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
-        writel(((u64) ioat_chan->completion_addr) >> 32,
-               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
-
-        tasklet_enable(&ioat_chan->cleanup_task);
-        ioat_dma_start_null_desc(ioat_chan);  /* give chain to dma device */
-        return ioat_chan->desccount;
-}
-
-/**
- * ioat_dma_free_chan_resources - release all the descriptors
- * @chan: the channel to be cleaned
- */
-static void ioat_dma_free_chan_resources(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
-        struct ioat_desc_sw *desc, *_desc;
-        int in_use_descs = 0;
-
-        /* Before freeing channel resources first check
-         * if they have been previously allocated for this channel.
-         */
-        if (ioat_chan->desccount == 0)
-                return;
-
-        tasklet_disable(&ioat_chan->cleanup_task);
-        ioat_dma_memcpy_cleanup(ioat_chan);
-
-        /* Delay 100ms after reset to allow internal DMA logic to quiesce
-         * before removing DMA descriptor resources.
-         */
-        writeb(IOAT_CHANCMD_RESET,
-               ioat_chan->reg_base
-                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-        mdelay(100);
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                list_for_each_entry_safe(desc, _desc,
-                                         &ioat_chan->used_desc, node) {
-                        in_use_descs++;
-                        list_del(&desc->node);
-                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                                      desc->async_tx.phys);
-                        kfree(desc);
-                }
-                list_for_each_entry_safe(desc, _desc,
-                                         &ioat_chan->free_desc, node) {
-                        list_del(&desc->node);
-                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                                      desc->async_tx.phys);
-                        kfree(desc);
-                }
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                list_for_each_entry_safe(desc, _desc,
-                                         ioat_chan->free_desc.next, node) {
-                        list_del(&desc->node);
-                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                                      desc->async_tx.phys);
-                        kfree(desc);
-                }
-                desc = to_ioat_desc(ioat_chan->free_desc.next);
-                pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                              desc->async_tx.phys);
-                kfree(desc);
-                INIT_LIST_HEAD(&ioat_chan->free_desc);
-                INIT_LIST_HEAD(&ioat_chan->used_desc);
-                break;
-        }
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        pci_pool_free(ioatdma_device->completion_pool,
-                      ioat_chan->completion_virt,
-                      ioat_chan->completion_addr);
-
-        /* one is ok since we left it on there on purpose */
-        if (in_use_descs > 1)
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "Freeing %d in use descriptors!\n",
-                        in_use_descs - 1);
-
-        ioat_chan->last_completion = ioat_chan->completion_addr = 0;
-        ioat_chan->pending = 0;
-        ioat_chan->dmacount = 0;
-        ioat_chan->desccount = 0;
-        ioat_chan->watchdog_completion = 0;
-        ioat_chan->last_compl_desc_addr_hw = 0;
-        ioat_chan->watchdog_tcp_cookie =
-                ioat_chan->watchdog_last_tcp_cookie = 0;
-}
-
-/**
- * ioat_dma_get_next_descriptor - return the next available descriptor
- * @ioat_chan: IOAT DMA channel handle
- *
- * Gets the next descriptor from the chain, and must be called with the
- * channel's desc_lock held.  Allocates more descriptors if the channel
- * has run out.
- */
-static struct ioat_desc_sw *
-ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *new;
-
-        if (!list_empty(&ioat_chan->free_desc)) {
-                new = to_ioat_desc(ioat_chan->free_desc.next);
-                list_del(&new->node);
-        } else {
-                /* try to get another desc */
-                new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
-                if (!new) {
-                        dev_err(&ioat_chan->device->pdev->dev,
-                                "alloc failed\n");
-                        return NULL;
-                }
-        }
-
-        prefetch(new->hw);
-        return new;
-}
-
-static struct ioat_desc_sw *
-ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *new;
-
-        /*
-         * used.prev points to where to start processing
-         * used.next points to next free descriptor
-         * if used.prev == NULL, there are none waiting to be processed
-         * if used.next == used.prev.prev, there is only one free descriptor,
-         *      and we need to use it to as a noop descriptor before
-         *      linking in a new set of descriptors, since the device
-         *      has probably already read the pointer to it
-         */
-        if (ioat_chan->used_desc.prev &&
-            ioat_chan->used_desc.next == ioat_chan->used_desc.prev->prev) {
-
-                struct ioat_desc_sw *desc;
-                struct ioat_desc_sw *noop_desc;
-                int i;
-
-                /* set up the noop descriptor */
-                noop_desc = to_ioat_desc(ioat_chan->used_desc.next);
-                /* set size to non-zero value (channel returns error when size is 0) */
-                noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
-                noop_desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
-                noop_desc->hw->src_addr = 0;
-                noop_desc->hw->dst_addr = 0;
-
-                ioat_chan->used_desc.next = ioat_chan->used_desc.next->next;
-                ioat_chan->pending++;
-                ioat_chan->dmacount++;
-
-                /* try to get a few more descriptors */
-                for (i = 16; i; i--) {
-                        desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
-                        if (!desc) {
-                                dev_err(&ioat_chan->device->pdev->dev,
-                                        "alloc failed\n");
-                                break;
-                        }
-                        list_add_tail(&desc->node, ioat_chan->used_desc.next);
-
-                        desc->hw->next
-                                = to_ioat_desc(desc->node.next)->async_tx.phys;
-                        to_ioat_desc(desc->node.prev)->hw->next
-                                = desc->async_tx.phys;
-                        ioat_chan->desccount++;
-                }
-
-                ioat_chan->used_desc.next = noop_desc->node.next;
-        }
-        new = to_ioat_desc(ioat_chan->used_desc.next);
-        prefetch(new);
-        ioat_chan->used_desc.next = new->node.next;
-
-        if (ioat_chan->used_desc.prev == NULL)
-                ioat_chan->used_desc.prev = &new->node;
-
-        prefetch(new->hw);
-        return new;
-}
-
-static struct ioat_desc_sw *ioat_dma_get_next_descriptor(
-                                                struct ioat_dma_chan *ioat_chan)
-{
-        if (!ioat_chan)
-                return NULL;
-
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                return ioat1_dma_get_next_descriptor(ioat_chan);
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                return ioat2_dma_get_next_descriptor(ioat_chan);
-        }
-        return NULL;
-}
-
-static struct dma_async_tx_descriptor *ioat1_dma_prep_memcpy(
-                                                struct dma_chan *chan,
-                                                dma_addr_t dma_dest,
-                                                dma_addr_t dma_src,
-                                                size_t len,
-                                                unsigned long flags)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioat_desc_sw *new;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        new = ioat_dma_get_next_descriptor(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        if (new) {
-                new->len = len;
-                new->dst = dma_dest;
-                new->src = dma_src;
-                new->async_tx.flags = flags;
-                return &new->async_tx;
-        } else {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
-                        chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
-                return NULL;
-        }
-}
-
-static struct dma_async_tx_descriptor *ioat2_dma_prep_memcpy(
-                                                struct dma_chan *chan,
-                                                dma_addr_t dma_dest,
-                                                dma_addr_t dma_src,
-                                                size_t len,
-                                                unsigned long flags)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioat_desc_sw *new;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        new = ioat2_dma_get_next_descriptor(ioat_chan);
-
-        /*
-         * leave ioat_chan->desc_lock set in ioat 2 path
-         * it will get unlocked at end of tx_submit
-         */
-
-        if (new) {
-                new->len = len;
-                new->dst = dma_dest;
-                new->src = dma_src;
-                new->async_tx.flags = flags;
-                return &new->async_tx;
-        } else {
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
-                        chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
-                return NULL;
-        }
-}
-
-static void ioat_dma_cleanup_tasklet(unsigned long data)
-{
-        struct ioat_dma_chan *chan = (void *)data;
-        ioat_dma_memcpy_cleanup(chan);
-        writew(IOAT_CHANCTRL_INT_DISABLE,
-               chan->reg_base + IOAT_CHANCTRL_OFFSET);
-}
-
-static void
-ioat_dma_unmap(struct ioat_dma_chan *ioat_chan, struct ioat_desc_sw *desc)
-{
-        if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-                if (desc->async_tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
-                        pci_unmap_single(ioat_chan->device->pdev,
-                                         pci_unmap_addr(desc, dst),
-                                         pci_unmap_len(desc, len),
-                                         PCI_DMA_FROMDEVICE);
-                else
-                        pci_unmap_page(ioat_chan->device->pdev,
-                                       pci_unmap_addr(desc, dst),
-                                       pci_unmap_len(desc, len),
-                                       PCI_DMA_FROMDEVICE);
-        }
-
-        if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-                if (desc->async_tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
-                        pci_unmap_single(ioat_chan->device->pdev,
-                                         pci_unmap_addr(desc, src),
-                                         pci_unmap_len(desc, len),
-                                         PCI_DMA_TODEVICE);
-                else
-                        pci_unmap_page(ioat_chan->device->pdev,
-                                       pci_unmap_addr(desc, src),
-                                       pci_unmap_len(desc, len),
-                                       PCI_DMA_TODEVICE);
-        }
-}
-
-/**
- * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
- * @chan: ioat channel to be cleaned up
- */
-static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
-{
-        unsigned long phys_complete;
-        struct ioat_desc_sw *desc, *_desc;
-        dma_cookie_t cookie = 0;
-        unsigned long desc_phys;
-        struct ioat_desc_sw *latest_desc;
-
-        prefetch(ioat_chan->completion_virt);
-
-        if (!spin_trylock_bh(&ioat_chan->cleanup_lock))
-                return;
-
-        /* The completion writeback can happen at any time,
-           so reads by the driver need to be atomic operations
-           The descriptor physical addresses are limited to 32-bits
-           when the CPU can only do a 32-bit mov */
-
-#if (BITS_PER_LONG == 64)
-        phys_complete =
-                ioat_chan->completion_virt->full
-                & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
-        phys_complete =
-                ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
-#endif
-
-        if ((ioat_chan->completion_virt->full
-                & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
-                                IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "Channel halted, chanerr = %x\n",
-                        readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
-
-                /* TODO do something to salvage the situation */
-        }
-
-        if (phys_complete == ioat_chan->last_completion) {
-                spin_unlock_bh(&ioat_chan->cleanup_lock);
-                /*
-                 * perhaps we're stuck so hard that the watchdog can't go off?
-                 * try to catch it after 2 seconds
-                 */
-                if (ioat_chan->device->version != IOAT_VER_3_0) {
-                        if (time_after(jiffies,
-                                       ioat_chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
-                                ioat_dma_chan_watchdog(&(ioat_chan->device->work.work));
-                                ioat_chan->last_completion_time = jiffies;
-                        }
-                }
-                return;
-        }
-        ioat_chan->last_completion_time = jiffies;
-
-        cookie = 0;
-        if (!spin_trylock_bh(&ioat_chan->desc_lock)) {
-                spin_unlock_bh(&ioat_chan->cleanup_lock);
-                return;
-        }
-
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                list_for_each_entry_safe(desc, _desc,
-                                         &ioat_chan->used_desc, node) {
-
-                        /*
-                         * Incoming DMA requests may use multiple descriptors,
-                         * due to exceeding xfercap, perhaps. If so, only the
-                         * last one will have a cookie, and require unmapping.
-                         */
-                        if (desc->async_tx.cookie) {
-                                cookie = desc->async_tx.cookie;
-                                ioat_dma_unmap(ioat_chan, desc);
-                                if (desc->async_tx.callback) {
-                                        desc->async_tx.callback(desc->async_tx.callback_param);
-                                        desc->async_tx.callback = NULL;
-                                }
-                        }
-
-                        if (desc->async_tx.phys != phys_complete) {
-                                /*
-                                 * a completed entry, but not the last, so clean
-                                 * up if the client is done with the descriptor
-                                 */
-                                if (async_tx_test_ack(&desc->async_tx)) {
-                                        list_move_tail(&desc->node,
-                                                       &ioat_chan->free_desc);
-                                } else
-                                        desc->async_tx.cookie = 0;
-                        } else {
-                                /*
-                                 * last used desc. Do not remove, so we can
-                                 * append from it, but don't look at it next
-                                 * time, either
-                                 */
-                                desc->async_tx.cookie = 0;
-
-                                /* TODO check status bits? */
-                                break;
-                        }
-                }
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                /* has some other thread has already cleaned up? */
-                if (ioat_chan->used_desc.prev == NULL)
-                        break;
-
-                /* work backwards to find latest finished desc */
-                desc = to_ioat_desc(ioat_chan->used_desc.next);
-                latest_desc = NULL;
-                do {
-                        desc = to_ioat_desc(desc->node.prev);
-                        desc_phys = (unsigned long)desc->async_tx.phys
-                                       & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-                        if (desc_phys == phys_complete) {
-                                latest_desc = desc;
-                                break;
-                        }
-                } while (&desc->node != ioat_chan->used_desc.prev);
-
-                if (latest_desc != NULL) {
-
-                        /* work forwards to clear finished descriptors */
-                        for (desc = to_ioat_desc(ioat_chan->used_desc.prev);
-                             &desc->node != latest_desc->node.next &&
-                             &desc->node != ioat_chan->used_desc.next;
-                             desc = to_ioat_desc(desc->node.next)) {
-                                if (desc->async_tx.cookie) {
-                                        cookie = desc->async_tx.cookie;
-                                        desc->async_tx.cookie = 0;
-                                        ioat_dma_unmap(ioat_chan, desc);
-                                        if (desc->async_tx.callback) {
-                                                desc->async_tx.callback(desc->async_tx.callback_param);
-                                                desc->async_tx.callback = NULL;
-                                        }
-                                }
-                        }
-
-                        /* move used.prev up beyond those that are finished */
-                        if (&desc->node == ioat_chan->used_desc.next)
-                                ioat_chan->used_desc.prev = NULL;
-                        else
-                                ioat_chan->used_desc.prev = &desc->node;
-                }
-                break;
-        }
-
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        ioat_chan->last_completion = phys_complete;
-        if (cookie != 0)
-                ioat_chan->completed_cookie = cookie;
-
-        spin_unlock_bh(&ioat_chan->cleanup_lock);
-}
-
-/**
- * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
- * @chan: IOAT DMA channel handle
- * @cookie: DMA transaction identifier
- * @done: if not %NULL, updated with last completed transaction
- * @used: if not %NULL, updated with last used transaction
- */
-static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
-                                            dma_cookie_t cookie,
-                                            dma_cookie_t *done,
-                                            dma_cookie_t *used)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        dma_cookie_t last_used;
-        dma_cookie_t last_complete;
-        enum dma_status ret;
-
-        last_used = chan->cookie;
-        last_complete = ioat_chan->completed_cookie;
-        ioat_chan->watchdog_tcp_cookie = cookie;
-
-        if (done)
-                *done = last_complete;
-        if (used)
-                *used = last_used;
-
-        ret = dma_async_is_complete(cookie, last_complete, last_used);
-        if (ret == DMA_SUCCESS)
-                return ret;
-
-        ioat_dma_memcpy_cleanup(ioat_chan);
-
-        last_used = chan->cookie;
-        last_complete = ioat_chan->completed_cookie;
-
-        if (done)
-                *done = last_complete;
-        if (used)
-                *used = last_used;
-
-        return dma_async_is_complete(cookie, last_complete, last_used);
-}
-
-static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *desc;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-
-        desc = ioat_dma_get_next_descriptor(ioat_chan);
-
-        if (!desc) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "Unable to start null desc - get next desc failed\n");
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                return;
-        }
-
-        desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL
-                                | IOAT_DMA_DESCRIPTOR_CTL_INT_GN
-                                | IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
-        /* set size to non-zero value (channel returns error when size is 0) */
-        desc->hw->size = NULL_DESC_BUFFER_SIZE;
-        desc->hw->src_addr = 0;
-        desc->hw->dst_addr = 0;
-        async_tx_ack(&desc->async_tx);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                desc->hw->next = 0;
-                list_add_tail(&desc->node, &ioat_chan->used_desc);
-
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
-                writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
-                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
-
-                ioat_chan->dmacount++;
-                __ioat2_dma_memcpy_issue_pending(ioat_chan);
-                break;
-        }
-        spin_unlock_bh(&ioat_chan->desc_lock);
-}
-
-/*
- * Perform a IOAT transaction to verify the HW works.
- */
-#define IOAT_TEST_SIZE 2000
-
-static void ioat_dma_test_callback(void *dma_async_param)
-{
-        struct completion *cmp = dma_async_param;
-
-        complete(cmp);
-}
-
-/**
- * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
- * @device: device to be tested
- */
-static int ioat_dma_self_test(struct ioatdma_device *device)
-{
-        int i;
-        u8 *src;
-        u8 *dest;
-        struct dma_chan *dma_chan;
-        struct dma_async_tx_descriptor *tx;
-        dma_addr_t dma_dest, dma_src;
-        dma_cookie_t cookie;
-        int err = 0;
-        struct completion cmp;
-        unsigned long tmo;
-        unsigned long flags;
-
-        src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
-        if (!src)
-                return -ENOMEM;
-        dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
-        if (!dest) {
-                kfree(src);
-                return -ENOMEM;
-        }
-
-        /* Fill in src buffer */
-        for (i = 0; i < IOAT_TEST_SIZE; i++)
-                src[i] = (u8)i;
-
-        /* Start copy, using first DMA channel */
-        dma_chan = container_of(device->common.channels.next,
-                                struct dma_chan,
-                                device_node);
-        if (device->common.device_alloc_chan_resources(dma_chan) < 1) {
-                dev_err(&device->pdev->dev,
-                        "selftest cannot allocate chan resource\n");
-                err = -ENODEV;
-                goto out;
-        }
-
-        dma_src = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
-                                 DMA_TO_DEVICE);
-        dma_dest = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
-                                  DMA_FROM_DEVICE);
-        flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE;
-        tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
-                                                   IOAT_TEST_SIZE, flags);
-        if (!tx) {
-                dev_err(&device->pdev->dev,
-                        "Self-test prep failed, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-
-        async_tx_ack(tx);
-        init_completion(&cmp);
-        tx->callback = ioat_dma_test_callback;
-        tx->callback_param = &cmp;
-        cookie = tx->tx_submit(tx);
-        if (cookie < 0) {
-                dev_err(&device->pdev->dev,
-                        "Self-test setup failed, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-        device->common.device_issue_pending(dma_chan);
-
-        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
-
-        if (tmo == 0 ||
-            device->common.device_is_tx_complete(dma_chan, cookie, NULL, NULL)
-                                        != DMA_SUCCESS) {
-                dev_err(&device->pdev->dev,
-                        "Self-test copy timed out, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-        if (memcmp(src, dest, IOAT_TEST_SIZE)) {
-                dev_err(&device->pdev->dev,
-                        "Self-test copy failed compare, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-
-free_resources:
-        device->common.device_free_chan_resources(dma_chan);
-out:
-        kfree(src);
-        kfree(dest);
-        return err;
-}
-
-static char ioat_interrupt_style[32] = "msix";
-module_param_string(ioat_interrupt_style, ioat_interrupt_style,
-                    sizeof(ioat_interrupt_style), 0644);
-MODULE_PARM_DESC(ioat_interrupt_style,
-                 "set ioat interrupt style: msix (default), "
-                 "msix-single-vector, msi, intx)");
-
-/**
- * ioat_dma_setup_interrupts - setup interrupt handler
- * @device: ioat device
- */
-static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
-{
-        struct ioat_dma_chan *ioat_chan;
-        int err, i, j, msixcnt;
-        u8 intrctrl = 0;
-
-        if (!strcmp(ioat_interrupt_style, "msix"))
-                goto msix;
-        if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
-                goto msix_single_vector;
-        if (!strcmp(ioat_interrupt_style, "msi"))
-                goto msi;
-        if (!strcmp(ioat_interrupt_style, "intx"))
-                goto intx;
-        dev_err(&device->pdev->dev, "invalid ioat_interrupt_style %s\n",
-                ioat_interrupt_style);
-        goto err_no_irq;
-
-msix:
-        /* The number of MSI-X vectors should equal the number of channels */
-        msixcnt = device->common.chancnt;
-        for (i = 0; i < msixcnt; i++)
-                device->msix_entries[i].entry = i;
-
-        err = pci_enable_msix(device->pdev, device->msix_entries, msixcnt);
-        if (err < 0)
-                goto msi;
-        if (err > 0)
-                goto msix_single_vector;
-
-        for (i = 0; i < msixcnt; i++) {
-                ioat_chan = ioat_lookup_chan_by_index(device, i);
-                err = request_irq(device->msix_entries[i].vector,
-                                  ioat_dma_do_interrupt_msix,
-                                  0, "ioat-msix", ioat_chan);
-                if (err) {
-                        for (j = 0; j < i; j++) {
-                                ioat_chan =
-                                        ioat_lookup_chan_by_index(device, j);
-                                free_irq(device->msix_entries[j].vector,
-                                         ioat_chan);
-                        }
-                        goto msix_single_vector;
-                }
-        }
-        intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
-        device->irq_mode = msix_multi_vector;
-        goto done;
-
-msix_single_vector:
-        device->msix_entries[0].entry = 0;
-        err = pci_enable_msix(device->pdev, device->msix_entries, 1);
-        if (err)
-                goto msi;
-
-        err = request_irq(device->msix_entries[0].vector, ioat_dma_do_interrupt,
-                          0, "ioat-msix", device);
-        if (err) {
-                pci_disable_msix(device->pdev);
-                goto msi;
-        }
-        device->irq_mode = msix_single_vector;
-        goto done;
-
-msi:
-        err = pci_enable_msi(device->pdev);
-        if (err)
-                goto intx;
-
-        err = request_irq(device->pdev->irq, ioat_dma_do_interrupt,
-                          0, "ioat-msi", device);
-        if (err) {
-                pci_disable_msi(device->pdev);
-                goto intx;
-        }
-        /*
-         * CB 1.2 devices need a bit set in configuration space to enable MSI
-         */
-        if (device->version == IOAT_VER_1_2) {
-                u32 dmactrl;
-                pci_read_config_dword(device->pdev,
-                                      IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
-                dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
-                pci_write_config_dword(device->pdev,
-                                       IOAT_PCI_DMACTRL_OFFSET, dmactrl);
-        }
-        device->irq_mode = msi;
-        goto done;
-
-intx:
-        err = request_irq(device->pdev->irq, ioat_dma_do_interrupt,
-                          IRQF_SHARED, "ioat-intx", device);
-        if (err)
-                goto err_no_irq;
-        device->irq_mode = intx;
-
-done:
-        intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
-        writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
-        return 0;
-
-err_no_irq:
-        /* Disable all interrupt generation */
-        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
-        dev_err(&device->pdev->dev, "no usable interrupts\n");
-        device->irq_mode = none;
-        return -1;
-}
-
-/**
- * ioat_dma_remove_interrupts - remove whatever interrupts were set
- * @device: ioat device
- */
-static void ioat_dma_remove_interrupts(struct ioatdma_device *device)
-{
-        struct ioat_dma_chan *ioat_chan;
-        int i;
-
-        /* Disable all interrupt generation */
-        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
-
-        switch (device->irq_mode) {
-        case msix_multi_vector:
-                for (i = 0; i < device->common.chancnt; i++) {
-                        ioat_chan = ioat_lookup_chan_by_index(device, i);
-                        free_irq(device->msix_entries[i].vector, ioat_chan);
-                }
-                pci_disable_msix(device->pdev);
-                break;
-        case msix_single_vector:
-                free_irq(device->msix_entries[0].vector, device);
-                pci_disable_msix(device->pdev);
-                break;
-        case msi:
-                free_irq(device->pdev->irq, device);
-                pci_disable_msi(device->pdev);
-                break;
-        case intx:
-                free_irq(device->pdev->irq, device);
-                break;
-        case none:
-                dev_warn(&device->pdev->dev,
-                         "call to %s without interrupts setup\n", __func__);
-        }
-        device->irq_mode = none;
-}
-
-struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
-                                      void __iomem *iobase)
-{
-        int err;
-        struct ioatdma_device *device;
-
-        device = kzalloc(sizeof(*device), GFP_KERNEL);
-        if (!device) {
-                err = -ENOMEM;
-                goto err_kzalloc;
-        }
-        device->pdev = pdev;
-        device->reg_base = iobase;
-        device->version = readb(device->reg_base + IOAT_VER_OFFSET);
-
-        /* DMA coherent memory pool for DMA descriptor allocations */
-        device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
-                                           sizeof(struct ioat_dma_descriptor),
-                                           64, 0);
-        if (!device->dma_pool) {
-                err = -ENOMEM;
-                goto err_dma_pool;
-        }
-
-        device->completion_pool = pci_pool_create("completion_pool", pdev,
-                                                  sizeof(u64), SMP_CACHE_BYTES,
-                                                  SMP_CACHE_BYTES);
-        if (!device->completion_pool) {
-                err = -ENOMEM;
-                goto err_completion_pool;
-        }
-
-        INIT_LIST_HEAD(&device->common.channels);
-        ioat_dma_enumerate_channels(device);
-
-        device->common.device_alloc_chan_resources =
-                                                ioat_dma_alloc_chan_resources;
-        device->common.device_free_chan_resources =
-                                                ioat_dma_free_chan_resources;
-        device->common.dev = &pdev->dev;
-
-        dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
-        device->common.device_is_tx_complete = ioat_dma_is_complete;
-        switch (device->version) {
-        case IOAT_VER_1_2:
-                device->common.device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
-                device->common.device_issue_pending =
-                                                ioat1_dma_memcpy_issue_pending;
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                device->common.device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
-                device->common.device_issue_pending =
-                                                ioat2_dma_memcpy_issue_pending;
-                break;
-        }
-
-        dev_err(&device->pdev->dev,
-                "Intel(R) I/OAT DMA Engine found,"
-                " %d channels, device version 0x%02x, driver version %s\n",
-                device->common.chancnt, device->version, IOAT_DMA_VERSION);
-
-        if (!device->common.chancnt) {
-                dev_err(&device->pdev->dev,
-                        "Intel(R) I/OAT DMA Engine problem found: "
-                        "zero channels detected\n");
-                goto err_setup_interrupts;
-        }
-
-        err = ioat_dma_setup_interrupts(device);
-        if (err)
-                goto err_setup_interrupts;
-
-        err = ioat_dma_self_test(device);
-        if (err)
-                goto err_self_test;
-
-        ioat_set_tcp_copy_break(device);
-
-        dma_async_device_register(&device->common);
-
-        if (device->version != IOAT_VER_3_0) {
-                INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
-                schedule_delayed_work(&device->work,
-                                      WATCHDOG_DELAY);
-        }
-
-        return device;
-
-err_self_test:
-        ioat_dma_remove_interrupts(device);
-err_setup_interrupts:
-        pci_pool_destroy(device->completion_pool);
-err_completion_pool:
-        pci_pool_destroy(device->dma_pool);
-err_dma_pool:
-        kfree(device);
-err_kzalloc:
-        dev_err(&pdev->dev,
-                "Intel(R) I/OAT DMA Engine initialization failed\n");
-        return NULL;
-}
-
-void ioat_dma_remove(struct ioatdma_device *device)
-{
-        struct dma_chan *chan, *_chan;
-        struct ioat_dma_chan *ioat_chan;
-
-        if (device->version != IOAT_VER_3_0)
-                cancel_delayed_work(&device->work);
-
-        ioat_dma_remove_interrupts(device);
-
-        dma_async_device_unregister(&device->common);
-
-        pci_pool_destroy(device->dma_pool);
-        pci_pool_destroy(device->completion_pool);
-
-        iounmap(device->reg_base);
-        pci_release_regions(device->pdev);
-        pci_disable_device(device->pdev);
-
-        list_for_each_entry_safe(chan, _chan,
-                                 &device->common.channels, device_node) {
-                ioat_chan = to_ioat_chan(chan);
-                list_del(&chan->device_node);
-                kfree(ioat_chan);
-        }
-        kfree(device);
-}
-
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h
deleted file mode 100644
index a52ff4bd4601..000000000000
--- a/drivers/dma/ioatdma.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59
- * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- *
- * The full GNU General Public License is included in this distribution in the
- * file called COPYING.
- */
-#ifndef IOATDMA_H
-#define IOATDMA_H
-
-#include <linux/dmaengine.h>
-#include "ioatdma_hw.h"
-#include <linux/init.h>
-#include <linux/dmapool.h>
-#include <linux/cache.h>
-#include <linux/pci_ids.h>
-#include <net/tcp.h>
-
-#define IOAT_DMA_VERSION  "3.64"
-
-enum ioat_interrupt {
-        none = 0,
-        msix_multi_vector = 1,
-        msix_single_vector = 2,
-        msi = 3,
-        intx = 4,
-};
-
-#define IOAT_LOW_COMPLETION_MASK        0xffffffc0
-#define IOAT_DMA_DCA_ANY_CPU            ~0
-#define IOAT_WATCHDOG_PERIOD            (2 * HZ)
-
-
-/**
- * struct ioatdma_device - internal representation of a IOAT device
- * @pdev: PCI-Express device
- * @reg_base: MMIO register space base address
- * @dma_pool: for allocating DMA descriptors
- * @common: embedded struct dma_device
- * @version: version of ioatdma device
- * @irq_mode: which style irq to use
- * @msix_entries: irq handlers
- * @idx: per channel data
- */
-
-struct ioatdma_device {
-        struct pci_dev *pdev;
-        void __iomem *reg_base;
-        struct pci_pool *dma_pool;
-        struct pci_pool *completion_pool;
-        struct dma_device common;
-        u8 version;
-        enum ioat_interrupt irq_mode;
-        struct delayed_work work;
-        struct msix_entry msix_entries[4];
-        struct ioat_dma_chan *idx[4];
-};
-
-/**
- * struct ioat_dma_chan - internal representation of a DMA channel
- */
-struct ioat_dma_chan {
-
-        void __iomem *reg_base;
-
-        dma_cookie_t completed_cookie;
-        unsigned long last_completion;
-        unsigned long last_completion_time;
-
-        size_t xfercap; /* XFERCAP register value expanded out */
-
-        spinlock_t cleanup_lock;
-        spinlock_t desc_lock;
-        struct list_head free_desc;
-        struct list_head used_desc;
-        unsigned long watchdog_completion;
-        int watchdog_tcp_cookie;
-        u32 watchdog_last_tcp_cookie;
-        struct delayed_work work;
-
-        int pending;
-        int dmacount;
-        int desccount;
-
-        struct ioatdma_device *device;
-        struct dma_chan common;
-
-        dma_addr_t completion_addr;
-        union {
-                u64 full; /* HW completion writeback */
-                struct {
-                        u32 low;
-                        u32 high;
-                };
-        } *completion_virt;
-        unsigned long last_compl_desc_addr_hw;
-        struct tasklet_struct cleanup_task;
-};
-
-/* wrapper around hardware descriptor format + additional software fields */
-
-/**
- * struct ioat_desc_sw - wrapper around hardware descriptor
- * @hw: hardware DMA descriptor
- * @node: this descriptor will either be on the free list,
- *     or attached to a transaction list (async_tx.tx_list)
- * @tx_cnt: number of descriptors required to complete the transaction
- * @async_tx: the generic software descriptor for all engines
- */
-struct ioat_desc_sw {
-        struct ioat_dma_descriptor *hw;
-        struct list_head node;
-        int tx_cnt;
-        size_t len;
-        dma_addr_t src;
-        dma_addr_t dst;
-        struct dma_async_tx_descriptor async_tx;
-};
-
-static inline void ioat_set_tcp_copy_break(struct ioatdma_device *dev)
-{
-        #ifdef CONFIG_NET_DMA
-        switch (dev->version) {
-        case IOAT_VER_1_2:
-                sysctl_tcp_dma_copybreak = 4096;
-                break;
-        case IOAT_VER_2_0:
-                sysctl_tcp_dma_copybreak = 2048;
-                break;
-        case IOAT_VER_3_0:
-                sysctl_tcp_dma_copybreak = 262144;
-                break;
-        }
-        #endif
-}
-
-#if defined(CONFIG_INTEL_IOATDMA) || defined(CONFIG_INTEL_IOATDMA_MODULE)
-struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
-                                      void __iomem *iobase);
-void ioat_dma_remove(struct ioatdma_device *device);
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-#else
-#define ioat_dma_probe(pdev, iobase)    NULL
-#define ioat_dma_remove(device)         do { } while (0)
-#define ioat_dca_init(pdev, iobase)     NULL
-#define ioat2_dca_init(pdev, iobase)    NULL
-#define ioat3_dca_init(pdev, iobase)    NULL
-#endif
-
-#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioatdma_hw.h b/drivers/dma/ioatdma_hw.h
deleted file mode 100644
index afa57eef86c9..000000000000
--- a/drivers/dma/ioatdma_hw.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59
- * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- *
- * The full GNU General Public License is included in this distribution in the
- * file called COPYING.
- */
-#ifndef _IOAT_HW_H_
-#define _IOAT_HW_H_
-
-/* PCI Configuration Space Values */
-#define IOAT_PCI_VID            0x8086
-
-/* CB device ID's */
-#define IOAT_PCI_DID_5000       0x1A38
-#define IOAT_PCI_DID_CNB        0x360B
-#define IOAT_PCI_DID_SCNB       0x65FF
-#define IOAT_PCI_DID_SNB        0x402F
-
-#define IOAT_PCI_RID            0x00
-#define IOAT_PCI_SVID           0x8086
-#define IOAT_PCI_SID            0x8086
-#define IOAT_VER_1_2            0x12    /* Version 1.2 */
-#define IOAT_VER_2_0            0x20    /* Version 2.0 */
-#define IOAT_VER_3_0            0x30    /* Version 3.0 */
-
-struct ioat_dma_descriptor {
-        uint32_t        size;
-        uint32_t        ctl;
-        uint64_t        src_addr;
-        uint64_t        dst_addr;
-        uint64_t        next;
-        uint64_t        rsv1;
-        uint64_t        rsv2;
-        uint64_t        user1;
-        uint64_t        user2;
-};
-
-#define IOAT_DMA_DESCRIPTOR_CTL_INT_GN  0x00000001
-#define IOAT_DMA_DESCRIPTOR_CTL_SRC_SN  0x00000002
-#define IOAT_DMA_DESCRIPTOR_CTL_DST_SN  0x00000004
-#define IOAT_DMA_DESCRIPTOR_CTL_CP_STS  0x00000008
-#define IOAT_DMA_DESCRIPTOR_CTL_FRAME   0x00000010
-#define IOAT_DMA_DESCRIPTOR_NUL         0x00000020
-#define IOAT_DMA_DESCRIPTOR_CTL_SP_BRK  0x00000040
-#define IOAT_DMA_DESCRIPTOR_CTL_DP_BRK  0x00000080
-#define IOAT_DMA_DESCRIPTOR_CTL_BNDL    0x00000100
-#define IOAT_DMA_DESCRIPTOR_CTL_DCA     0x00000200
-#define IOAT_DMA_DESCRIPTOR_CTL_BUFHINT 0x00000400
-
-#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_CONTEXT  0xFF000000
-#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_DMA      0x00000000
-
-#define IOAT_DMA_DESCRIPTOR_CTL_CONTEXT_DCA     0x00000001
-#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_MASK     0xFF000000
-
-#endif
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 2f052265122f..645ca8d54ec4 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -31,6 +31,7 @@
 #include <linux/platform_device.h>
 #include <linux/memory.h>
 #include <linux/ioport.h>
+#include <linux/raid/pq.h>
 
 #include <mach/adma.h>
 
@@ -57,65 +58,110 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
         }
 }
 
+static void
+iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
+{
+        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+        struct iop_adma_desc_slot *unmap = desc->group_head;
+        struct device *dev = &iop_chan->device->pdev->dev;
+        u32 len = unmap->unmap_len;
+        enum dma_ctrl_flags flags = tx->flags;
+        u32 src_cnt;
+        dma_addr_t addr;
+        dma_addr_t dest;
+
+        src_cnt = unmap->unmap_src_cnt;
+        dest = iop_desc_get_dest_addr(unmap, iop_chan);
+        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                enum dma_data_direction dir;
+
+                if (src_cnt > 1) /* is xor? */
+                        dir = DMA_BIDIRECTIONAL;
+                else
+                        dir = DMA_FROM_DEVICE;
+
+                dma_unmap_page(dev, dest, len, dir);
+        }
+
+        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                while (src_cnt--) {
+                        addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
+                        if (addr == dest)
+                                continue;
+                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
+                }
+        }
+        desc->group_head = NULL;
+}
+
+static void
+iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
+{
+        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+        struct iop_adma_desc_slot *unmap = desc->group_head;
+        struct device *dev = &iop_chan->device->pdev->dev;
+        u32 len = unmap->unmap_len;
+        enum dma_ctrl_flags flags = tx->flags;
+        u32 src_cnt = unmap->unmap_src_cnt;
+        dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
+        dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
+        int i;
+
+        if (tx->flags & DMA_PREP_CONTINUE)
+                src_cnt -= 3;
+
+        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
+                dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
+                dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
+        }
+
+        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                dma_addr_t addr;
+
+                for (i = 0; i < src_cnt; i++) {
+                        addr = iop_desc_get_src_addr(unmap, iop_chan, i);
+                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
+                }
+                if (desc->pq_check_result) {
+                        dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
+                        dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
+                }
+        }
+
+        desc->group_head = NULL;
+}
+
+
 static dma_cookie_t
 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 {
-        BUG_ON(desc->async_tx.cookie < 0);
-        if (desc->async_tx.cookie > 0) {
-                cookie = desc->async_tx.cookie;
-                desc->async_tx.cookie = 0;
+        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+
+        BUG_ON(tx->cookie < 0);
+        if (tx->cookie > 0) {
+                cookie = tx->cookie;
+                tx->cookie = 0;
 
                 /* call the callback (must not sleep or submit new
                  * operations to this channel)
                  */
-                if (desc->async_tx.callback)
-                        desc->async_tx.callback(
-                                desc->async_tx.callback_param);
+                if (tx->callback)
+                        tx->callback(tx->callback_param);
 
                 /* unmap dma addresses
                  * (unmap_single vs unmap_page?)
                  */
                 if (desc->group_head && desc->unmap_len) {
-                        struct iop_adma_desc_slot *unmap = desc->group_head;
-                        struct device *dev =
-                                &iop_chan->device->pdev->dev;
-                        u32 len = unmap->unmap_len;
-                        enum dma_ctrl_flags flags = desc->async_tx.flags;
-                        u32 src_cnt;
-                        dma_addr_t addr;
-                        dma_addr_t dest;
-
-                        src_cnt = unmap->unmap_src_cnt;
-                        dest = iop_desc_get_dest_addr(unmap, iop_chan);
-                        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-                                enum dma_data_direction dir;
-
-                                if (src_cnt > 1) /* is xor? */
-                                        dir = DMA_BIDIRECTIONAL;
-                                else
-                                        dir = DMA_FROM_DEVICE;
-
-                                dma_unmap_page(dev, dest, len, dir);
-                        }
-
-                        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-                                while (src_cnt--) {
-                                        addr = iop_desc_get_src_addr(unmap,
-                                                                     iop_chan,
-                                                                     src_cnt);
-                                        if (addr == dest)
-                                                continue;
-                                        dma_unmap_page(dev, addr, len,
-                                                       DMA_TO_DEVICE);
-                                }
-                        }
-                        desc->group_head = NULL;
+                        if (iop_desc_is_pq(desc))
+                                iop_desc_unmap_pq(iop_chan, desc);
+                        else
+                                iop_desc_unmap(iop_chan, desc);
                 }
         }
 
         /* run dependent operations */
-        dma_run_dependencies(&desc->async_tx);
+        dma_run_dependencies(tx);
 
         return cookie;
 }
@@ -287,7 +333,12 @@ static void iop_adma_tasklet(unsigned long data)
 {
         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 
-        spin_lock(&iop_chan->lock);
+        /* lockdep will flag depedency submissions as potentially
+         * recursive locking, this is not the case as a dependency
+         * submission will never recurse a channels submit routine.
+         * There are checks in async_tx.c to prevent this.
+         */
+        spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
         __iop_adma_slot_cleanup(iop_chan);
         spin_unlock(&iop_chan->lock);
 }
@@ -370,7 +421,7 @@ retry:
                         }
                         alloc_tail->group_head = alloc_start;
                         alloc_tail->async_tx.cookie = -EBUSY;
-                        list_splice(&chain, &alloc_tail->async_tx.tx_list);
+                        list_splice(&chain, &alloc_tail->tx_list);
                         iop_chan->last_used = last_used;
                         iop_desc_clear_next_desc(alloc_start);
                         iop_desc_clear_next_desc(alloc_tail);
@@ -429,7 +480,7 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 
         old_chain_tail = list_entry(iop_chan->chain.prev,
                 struct iop_adma_desc_slot, chain_node);
-        list_splice_init(&sw_desc->async_tx.tx_list,
+        list_splice_init(&sw_desc->tx_list,
                          &old_chain_tail->chain_node);
 
         /* fix up the hardware chain */
@@ -496,6 +547,7 @@ static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 
                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
                 slot->async_tx.tx_submit = iop_adma_tx_submit;
+                INIT_LIST_HEAD(&slot->tx_list);
                 INIT_LIST_HEAD(&slot->chain_node);
                 INIT_LIST_HEAD(&slot->slot_node);
                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
@@ -660,9 +712,9 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 }
 
 static struct dma_async_tx_descriptor *
-iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src,
-                           unsigned int src_cnt, size_t len, u32 *result,
-                           unsigned long flags)
+iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
+                          unsigned int src_cnt, size_t len, u32 *result,
+                          unsigned long flags)
 {
         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
         struct iop_adma_desc_slot *sw_desc, *grp_start;
@@ -696,6 +748,118 @@ iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src,
         return sw_desc ? &sw_desc->async_tx : NULL;
 }
 
+static struct dma_async_tx_descriptor *
+iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
+                     unsigned int src_cnt, const unsigned char *scf, size_t len,
+                     unsigned long flags)
+{
+        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
+        struct iop_adma_desc_slot *sw_desc, *g;
+        int slot_cnt, slots_per_op;
+        int continue_srcs;
+
+        if (unlikely(!len))
+                return NULL;
+        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
+
+        dev_dbg(iop_chan->device->common.dev,
+                "%s src_cnt: %d len: %u flags: %lx\n",
+                __func__, src_cnt, len, flags);
+
+        if (dmaf_p_disabled_continue(flags))
+                continue_srcs = 1+src_cnt;
+        else if (dmaf_continue(flags))
+                continue_srcs = 3+src_cnt;
+        else
+                continue_srcs = 0+src_cnt;
+
+        spin_lock_bh(&iop_chan->lock);
+        slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
+        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
+        if (sw_desc) {
+                int i;
+
+                g = sw_desc->group_head;
+                iop_desc_set_byte_count(g, iop_chan, len);
+
+                /* even if P is disabled its destination address (bits
+                 * [3:0]) must match Q.  It is ok if P points to an
+                 * invalid address, it won't be written.
+                 */
+                if (flags & DMA_PREP_PQ_DISABLE_P)
+                        dst[0] = dst[1] & 0x7;
+
+                iop_desc_set_pq_addr(g, dst);
+                sw_desc->unmap_src_cnt = src_cnt;
+                sw_desc->unmap_len = len;
+                sw_desc->async_tx.flags = flags;
+                for (i = 0; i < src_cnt; i++)
+                        iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
+
+                /* if we are continuing a previous operation factor in
+                 * the old p and q values, see the comment for dma_maxpq
+                 * in include/linux/dmaengine.h
+                 */
+                if (dmaf_p_disabled_continue(flags))
+                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
+                else if (dmaf_continue(flags)) {
+                        iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
+                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
+                        iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
+                }
+                iop_desc_init_pq(g, i, flags);
+        }
+        spin_unlock_bh(&iop_chan->lock);
+
+        return sw_desc ? &sw_desc->async_tx : NULL;
+}
+
+static struct dma_async_tx_descriptor *
+iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
+                         unsigned int src_cnt, const unsigned char *scf,
+                         size_t len, enum sum_check_flags *pqres,
+                         unsigned long flags)
+{
+        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
+        struct iop_adma_desc_slot *sw_desc, *g;
+        int slot_cnt, slots_per_op;
+
+        if (unlikely(!len))
+                return NULL;
+        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
+
+        dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
+                __func__, src_cnt, len);
+
+        spin_lock_bh(&iop_chan->lock);
+        slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
+        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
+        if (sw_desc) {
+                /* for validate operations p and q are tagged onto the
+                 * end of the source list
+                 */
+                int pq_idx = src_cnt;
+
+                g = sw_desc->group_head;
+                iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
+                iop_desc_set_pq_zero_sum_byte_count(g, len);
+                g->pq_check_result = pqres;
+                pr_debug("\t%s: g->pq_check_result: %p\n",
+                        __func__, g->pq_check_result);
+                sw_desc->unmap_src_cnt = src_cnt+2;
+                sw_desc->unmap_len = len;
+                sw_desc->async_tx.flags = flags;
+                while (src_cnt--)
+                        iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
+                                                          src[src_cnt],
+                                                          scf[src_cnt]);
+                iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
+        }
+        spin_unlock_bh(&iop_chan->lock);
+
+        return sw_desc ? &sw_desc->async_tx : NULL;
+}
+
 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 {
         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
@@ -906,7 +1070,7 @@ out:
 
 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
 static int __devinit
-iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
+iop_adma_xor_val_self_test(struct iop_adma_device *device)
 {
         int i, src_idx;
         struct page *dest;
@@ -1002,7 +1166,7 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
                 PAGE_SIZE, DMA_TO_DEVICE);
 
         /* skip zero sum if the capability is not present */
-        if (!dma_has_cap(DMA_ZERO_SUM, dma_chan->device->cap_mask))
+        if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
                 goto free_resources;
 
         /* zero sum the sources with the destintation page */
@@ -1016,10 +1180,10 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
                                            zero_sum_srcs[i], 0, PAGE_SIZE,
                                            DMA_TO_DEVICE);
-        tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
-                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
-                                        &zero_sum_result,
-                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
+                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                       &zero_sum_result,
+                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
         cookie = iop_adma_tx_submit(tx);
         iop_adma_issue_pending(dma_chan);
@@ -1072,10 +1236,10 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
                                            zero_sum_srcs[i], 0, PAGE_SIZE,
                                            DMA_TO_DEVICE);
-        tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
-                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
-                                        &zero_sum_result,
-                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
+                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                       &zero_sum_result,
+                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
         cookie = iop_adma_tx_submit(tx);
         iop_adma_issue_pending(dma_chan);
@@ -1105,6 +1269,170 @@ out:
         return err;
 }
 
+#ifdef CONFIG_MD_RAID6_PQ
+static int __devinit
+iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
+{
+        /* combined sources, software pq results, and extra hw pq results */
+        struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
+        /* ptr to the extra hw pq buffers defined above */
+        struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
+        /* address conversion buffers (dma_map / page_address) */
+        void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
+        dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST];
+        dma_addr_t pq_dest[2];
+
+        int i;
+        struct dma_async_tx_descriptor *tx;
+        struct dma_chan *dma_chan;
+        dma_cookie_t cookie;
+        u32 zero_sum_result;
+        int err = 0;
+        struct device *dev;
+
+        dev_dbg(device->common.dev, "%s\n", __func__);
+
+        for (i = 0; i < ARRAY_SIZE(pq); i++) {
+                pq[i] = alloc_page(GFP_KERNEL);
+                if (!pq[i]) {
+                        while (i--)
+                                __free_page(pq[i]);
+                        return -ENOMEM;
+                }
+        }
+
+        /* Fill in src buffers */
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
+                pq_sw[i] = page_address(pq[i]);
+                memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
+        }
+        pq_sw[i] = page_address(pq[i]);
+        pq_sw[i+1] = page_address(pq[i+1]);
+
+        dma_chan = container_of(device->common.channels.next,
+                                struct dma_chan,
+                                device_node);
+        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
+                err = -ENODEV;
+                goto out;
+        }
+
+        dev = dma_chan->device->dev;
+
+        /* initialize the dests */
+        memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
+        memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
+
+        /* test pq */
+        pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
+        pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
+                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
+                                         DMA_TO_DEVICE);
+
+        tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
+                                  IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
+                                  PAGE_SIZE,
+                                  DMA_PREP_INTERRUPT |
+                                  DMA_CTRL_ACK);
+
+        cookie = iop_adma_tx_submit(tx);
+        iop_adma_issue_pending(dma_chan);
+        msleep(8);
+
+        if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
+                DMA_SUCCESS) {
+                dev_err(dev, "Self-test pq timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
+
+        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
+                   page_address(pq_hw[0]), PAGE_SIZE) != 0) {
+                dev_err(dev, "Self-test p failed compare, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
+                   page_address(pq_hw[1]), PAGE_SIZE) != 0) {
+                dev_err(dev, "Self-test q failed compare, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        /* test correct zero sum using the software generated pq values */
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
+                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
+                                         DMA_TO_DEVICE);
+
+        zero_sum_result = ~0;
+        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
+                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
+                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
+                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
+
+        cookie = iop_adma_tx_submit(tx);
+        iop_adma_issue_pending(dma_chan);
+        msleep(8);
+
+        if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
+                DMA_SUCCESS) {
+                dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (zero_sum_result != 0) {
+                dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
+                        zero_sum_result);
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        /* test incorrect zero sum */
+        i = IOP_ADMA_NUM_SRC_TEST;
+        memset(pq_sw[i] + 100, 0, 100);
+        memset(pq_sw[i+1] + 200, 0, 200);
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
+                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
+                                         DMA_TO_DEVICE);
+
+        zero_sum_result = 0;
+        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
+                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
+                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
+                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
+
+        cookie = iop_adma_tx_submit(tx);
+        iop_adma_issue_pending(dma_chan);
+        msleep(8);
+
+        if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
+                DMA_SUCCESS) {
+                dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
+                dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
+                        zero_sum_result);
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+free_resources:
+        iop_adma_free_chan_resources(dma_chan);
+out:
+        i = ARRAY_SIZE(pq);
+        while (i--)
+                __free_page(pq[i]);
+        return err;
+}
+#endif
+
 static int __devexit iop_adma_remove(struct platform_device *dev)
 {
         struct iop_adma_device *device = platform_get_drvdata(dev);
@@ -1192,9 +1520,16 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
                 dma_dev->max_xor = iop_adma_get_max_xor();
                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
         }
-        if (dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask))
-                dma_dev->device_prep_dma_zero_sum =
-                        iop_adma_prep_dma_zero_sum;
+        if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
+                dma_dev->device_prep_dma_xor_val =
+                        iop_adma_prep_dma_xor_val;
+        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
+                dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
+                dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
+        }
+        if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
+                dma_dev->device_prep_dma_pq_val =
+                        iop_adma_prep_dma_pq_val;
         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
                 dma_dev->device_prep_dma_interrupt =
                         iop_adma_prep_dma_interrupt;
@@ -1248,23 +1583,35 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
         }
 
         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
-                dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
-                ret = iop_adma_xor_zero_sum_self_test(adev);
+            dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
+                ret = iop_adma_xor_val_self_test(adev);
                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
                 if (ret)
                         goto err_free_iop_chan;
         }
 
+        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
+            dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
+                #ifdef CONFIG_MD_RAID6_PQ
+                ret = iop_adma_pq_zero_sum_self_test(adev);
+                dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
+                #else
+                /* can not test raid6, so do not publish capability */
+                dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
+                dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
+                ret = 0;
+                #endif
+                if (ret)
+                        goto err_free_iop_chan;
+        }
+
         dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
-          "( %s%s%s%s%s%s%s%s%s%s)\n",
-          dma_has_cap(DMA_PQ_XOR, dma_dev->cap_mask) ? "pq_xor " : "",
-          dma_has_cap(DMA_PQ_UPDATE, dma_dev->cap_mask) ? "pq_update " : "",
-          dma_has_cap(DMA_PQ_ZERO_SUM, dma_dev->cap_mask) ? "pq_zero_sum " : "",
+          "( %s%s%s%s%s%s%s)\n",
+          dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
+          dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
           dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
-          dma_has_cap(DMA_DUAL_XOR, dma_dev->cap_mask) ? "dual_xor " : "",
-          dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask) ? "xor_zero_sum " : "",
+          dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
           dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)  ? "fill " : "",
-          dma_has_cap(DMA_MEMCPY_CRC32C, dma_dev->cap_mask) ? "cpy+crc " : "",
           dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
           dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
 
@@ -1296,7 +1643,7 @@ static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
         if (sw_desc) {
                 grp_start = sw_desc->group_head;
 
-                list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
                 async_tx_ack(&sw_desc->async_tx);
                 iop_desc_init_memcpy(grp_start, 0);
                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
@@ -1352,7 +1699,7 @@ static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
         if (sw_desc) {
                 grp_start = sw_desc->group_head;
-                list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
                 async_tx_ack(&sw_desc->async_tx);
                 iop_desc_init_null_xor(grp_start, 2, 0);
                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
diff --git a/drivers/dma/iovlock.c b/drivers/dma/iovlock.c
index 9f6fe46a9b87..c0a272c73682 100644
--- a/drivers/dma/iovlock.c
+++ b/drivers/dma/iovlock.c
@@ -183,6 +183,11 @@ dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
                                         iov_byte_offset,
                                         kdata,
                                         copy);
+                        /* poll for a descriptor slot */
+                        if (unlikely(dma_cookie < 0)) {
+                                dma_async_issue_pending(chan);
+                                continue;
+                        }
 
                         len -= copy;
                         iov[iovec_idx].iov_len -= copy;
@@ -248,6 +253,11 @@ dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
                                         page,
                                         offset,
                                         copy);
+                        /* poll for a descriptor slot */
+                        if (unlikely(dma_cookie < 0)) {
+                                dma_async_issue_pending(chan);
+                                continue;
+                        }
 
                         len -= copy;
                         iov[iovec_idx].iov_len -= copy;
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 3f23eabe09f2..466ab10c1ff1 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -517,7 +517,7 @@ retry:
                         }
                         alloc_tail->group_head = alloc_start;
                         alloc_tail->async_tx.cookie = -EBUSY;
-                        list_splice(&chain, &alloc_tail->async_tx.tx_list);
+                        list_splice(&chain, &alloc_tail->tx_list);
                         mv_chan->last_used = last_used;
                         mv_desc_clear_next_desc(alloc_start);
                         mv_desc_clear_next_desc(alloc_tail);
@@ -565,14 +565,14 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
         cookie = mv_desc_assign_cookie(mv_chan, sw_desc);
 
         if (list_empty(&mv_chan->chain))
-                list_splice_init(&sw_desc->async_tx.tx_list, &mv_chan->chain);
+                list_splice_init(&sw_desc->tx_list, &mv_chan->chain);
         else {
                 new_hw_chain = 0;
 
                 old_chain_tail = list_entry(mv_chan->chain.prev,
                                             struct mv_xor_desc_slot,
                                             chain_node);
-                list_splice_init(&grp_start->async_tx.tx_list,
+                list_splice_init(&grp_start->tx_list,
                                  &old_chain_tail->chain_node);
 
                 if (!mv_can_chain(grp_start))
@@ -632,6 +632,7 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
                 slot->async_tx.tx_submit = mv_xor_tx_submit;
                 INIT_LIST_HEAD(&slot->chain_node);
                 INIT_LIST_HEAD(&slot->slot_node);
+                INIT_LIST_HEAD(&slot->tx_list);
                 hw_desc = (char *) mv_chan->device->dma_desc_pool;
                 slot->async_tx.phys =
                         (dma_addr_t) &hw_desc[idx * MV_XOR_SLOT_SIZE];
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 06cafe1ef521..977b592e976b 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -126,9 +126,8 @@ struct mv_xor_chan {
  * @idx: pool index
  * @unmap_src_cnt: number of xor sources
  * @unmap_len: transaction bytecount
+ * @tx_list: list of slots that make up a multi-descriptor transaction
  * @async_tx: support for the async_tx api
- * @group_list: list of slots that make up a multi-descriptor transaction
- *      for example transfer lengths larger than the supported hw max
  * @xor_check_result: result of zero sum
  * @crc32_result: result crc calculation
  */
@@ -145,6 +144,7 @@ struct mv_xor_desc_slot {
         u16                     unmap_src_cnt;
         u32                     value;
         size_t                  unmap_len;
+        struct list_head        tx_list;
         struct dma_async_tx_descriptor  async_tx;
         union {
                 u32             *xor_check_result;
diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c
new file mode 100644
index 000000000000..b3b065c4e5c1
--- /dev/null
+++ b/drivers/dma/shdma.c
@@ -0,0 +1,786 @@
+/*
+ * Renesas SuperH DMA Engine support
+ *
+ * base is drivers/dma/flsdma.c
+ *
+ * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
+ * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
+ * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * - DMA of SuperH does not have Hardware DMA chain mode.
+ * - MAX DMA size is 16MB.
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/platform_device.h>
+#include <cpu/dma.h>
+#include <asm/dma-sh.h>
+#include "shdma.h"
+
+/* DMA descriptor control */
+#define DESC_LAST       (-1)
+#define DESC_COMP       (1)
+#define DESC_NCOMP      (0)
+
+#define NR_DESCS_PER_CHANNEL 32
+/*
+ * Define the default configuration for dual address memory-memory transfer.
+ * The 0x400 value represents auto-request, external->external.
+ *
+ * And this driver set 4byte burst mode.
+ * If you want to change mode, you need to change RS_DEFAULT of value.
+ * (ex 1byte burst mode -> (RS_DUAL & ~TS_32)
+ */
+#define RS_DEFAULT  (RS_DUAL)
+
+#define SH_DMAC_CHAN_BASE(id) (dma_base_addr[id])
+static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg)
+{
+        ctrl_outl(data, (SH_DMAC_CHAN_BASE(sh_dc->id) + reg));
+}
+
+static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg)
+{
+        return ctrl_inl((SH_DMAC_CHAN_BASE(sh_dc->id) + reg));
+}
+
+static void dmae_init(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = RS_DEFAULT; /* default is DUAL mode */
+        sh_dmae_writel(sh_chan, chcr, CHCR);
+}
+
+/*
+ * Reset DMA controller
+ *
+ * SH7780 has two DMAOR register
+ */
+static void sh_dmae_ctl_stop(int id)
+{
+        unsigned short dmaor = dmaor_read_reg(id);
+
+        dmaor &= ~(DMAOR_NMIF | DMAOR_AE);
+        dmaor_write_reg(id, dmaor);
+}
+
+static int sh_dmae_rst(int id)
+{
+        unsigned short dmaor;
+
+        sh_dmae_ctl_stop(id);
+        dmaor = (dmaor_read_reg(id)|DMAOR_INIT);
+
+        dmaor_write_reg(id, dmaor);
+        if ((dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF))) {
+                pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n");
+                return -EINVAL;
+        }
+        return 0;
+}
+
+static int dmae_is_idle(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+        if (chcr & CHCR_DE) {
+                if (!(chcr & CHCR_TE))
+                        return -EBUSY; /* working */
+        }
+        return 0; /* waiting */
+}
+
+static inline unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+        return ts_shift[(chcr & CHCR_TS_MASK) >> CHCR_TS_SHIFT];
+}
+
+static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs hw)
+{
+        sh_dmae_writel(sh_chan, hw.sar, SAR);
+        sh_dmae_writel(sh_chan, hw.dar, DAR);
+        sh_dmae_writel(sh_chan,
+                (hw.tcr >> calc_xmit_shift(sh_chan)), TCR);
+}
+
+static void dmae_start(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+
+        chcr |= (CHCR_DE|CHCR_IE);
+        sh_dmae_writel(sh_chan, chcr, CHCR);
+}
+
+static void dmae_halt(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+
+        chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE);
+        sh_dmae_writel(sh_chan, chcr, CHCR);
+}
+
+static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
+{
+        int ret = dmae_is_idle(sh_chan);
+        /* When DMA was working, can not set data to CHCR */
+        if (ret)
+                return ret;
+
+        sh_dmae_writel(sh_chan, val, CHCR);
+        return 0;
+}
+
+#define DMARS1_ADDR     0x04
+#define DMARS2_ADDR     0x08
+#define DMARS_SHIFT 8
+#define DMARS_CHAN_MSK 0x01
+static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
+{
+        u32 addr;
+        int shift = 0;
+        int ret = dmae_is_idle(sh_chan);
+        if (ret)
+                return ret;
+
+        if (sh_chan->id & DMARS_CHAN_MSK)
+                shift = DMARS_SHIFT;
+
+        switch (sh_chan->id) {
+        /* DMARS0 */
+        case 0:
+        case 1:
+                addr = SH_DMARS_BASE;
+                break;
+        /* DMARS1 */
+        case 2:
+        case 3:
+                addr = (SH_DMARS_BASE + DMARS1_ADDR);
+                break;
+        /* DMARS2 */
+        case 4:
+        case 5:
+                addr = (SH_DMARS_BASE + DMARS2_ADDR);
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        ctrl_outw((val << shift) |
+                (ctrl_inw(addr) & (shift ? 0xFF00 : 0x00FF)),
+                addr);
+
+        return 0;
+}
+
+static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+        struct sh_desc *desc = tx_to_sh_desc(tx);
+        struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan);
+        dma_cookie_t cookie;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+
+        cookie = sh_chan->common.cookie;
+        cookie++;
+        if (cookie < 0)
+                cookie = 1;
+
+        /* If desc only in the case of 1 */
+        if (desc->async_tx.cookie != -EBUSY)
+                desc->async_tx.cookie = cookie;
+        sh_chan->common.cookie = desc->async_tx.cookie;
+
+        list_splice_init(&desc->tx_list, sh_chan->ld_queue.prev);
+
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        return cookie;
+}
+
+static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan)
+{
+        struct sh_desc *desc, *_desc, *ret = NULL;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+        list_for_each_entry_safe(desc, _desc, &sh_chan->ld_free, node) {
+                if (async_tx_test_ack(&desc->async_tx)) {
+                        list_del(&desc->node);
+                        ret = desc;
+                        break;
+                }
+        }
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        return ret;
+}
+
+static void sh_dmae_put_desc(struct sh_dmae_chan *sh_chan, struct sh_desc *desc)
+{
+        if (desc) {
+                spin_lock_bh(&sh_chan->desc_lock);
+
+                list_splice_init(&desc->tx_list, &sh_chan->ld_free);
+                list_add(&desc->node, &sh_chan->ld_free);
+
+                spin_unlock_bh(&sh_chan->desc_lock);
+        }
+}
+
+static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        struct sh_desc *desc;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+        while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) {
+                spin_unlock_bh(&sh_chan->desc_lock);
+                desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL);
+                if (!desc) {
+                        spin_lock_bh(&sh_chan->desc_lock);
+                        break;
+                }
+                dma_async_tx_descriptor_init(&desc->async_tx,
+                                        &sh_chan->common);
+                desc->async_tx.tx_submit = sh_dmae_tx_submit;
+                desc->async_tx.flags = DMA_CTRL_ACK;
+                INIT_LIST_HEAD(&desc->tx_list);
+                sh_dmae_put_desc(sh_chan, desc);
+
+                spin_lock_bh(&sh_chan->desc_lock);
+                sh_chan->descs_allocated++;
+        }
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        return sh_chan->descs_allocated;
+}
+
+/*
+ * sh_dma_free_chan_resources - Free all resources of the channel.
+ */
+static void sh_dmae_free_chan_resources(struct dma_chan *chan)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        struct sh_desc *desc, *_desc;
+        LIST_HEAD(list);
+
+        BUG_ON(!list_empty(&sh_chan->ld_queue));
+        spin_lock_bh(&sh_chan->desc_lock);
+
+        list_splice_init(&sh_chan->ld_free, &list);
+        sh_chan->descs_allocated = 0;
+
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        list_for_each_entry_safe(desc, _desc, &list, node)
+                kfree(desc);
+}
+
+static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
+        struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src,
+        size_t len, unsigned long flags)
+{
+        struct sh_dmae_chan *sh_chan;
+        struct sh_desc *first = NULL, *prev = NULL, *new;
+        size_t copy_size;
+
+        if (!chan)
+                return NULL;
+
+        if (!len)
+                return NULL;
+
+        sh_chan = to_sh_chan(chan);
+
+        do {
+                /* Allocate the link descriptor from DMA pool */
+                new = sh_dmae_get_desc(sh_chan);
+                if (!new) {
+                        dev_err(sh_chan->dev,
+                                        "No free memory for link descriptor\n");
+                        goto err_get_desc;
+                }
+
+                copy_size = min(len, (size_t)SH_DMA_TCR_MAX);
+
+                new->hw.sar = dma_src;
+                new->hw.dar = dma_dest;
+                new->hw.tcr = copy_size;
+                if (!first)
+                        first = new;
+
+                new->mark = DESC_NCOMP;
+                async_tx_ack(&new->async_tx);
+
+                prev = new;
+                len -= copy_size;
+                dma_src += copy_size;
+                dma_dest += copy_size;
+                /* Insert the link descriptor to the LD ring */
+                list_add_tail(&new->node, &first->tx_list);
+        } while (len);
+
+        new->async_tx.flags = flags; /* client is in control of this ack */
+        new->async_tx.cookie = -EBUSY; /* Last desc */
+
+        return &first->async_tx;
+
+err_get_desc:
+        sh_dmae_put_desc(sh_chan, first);
+        return NULL;
+
+}
+
+/*
+ * sh_chan_ld_cleanup - Clean up link descriptors
+ *
+ * This function clean up the ld_queue of DMA channel.
+ */
+static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan)
+{
+        struct sh_desc *desc, *_desc;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+        list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) {
+                dma_async_tx_callback callback;
+                void *callback_param;
+
+                /* non send data */
+                if (desc->mark == DESC_NCOMP)
+                        break;
+
+                /* send data sesc */
+                callback = desc->async_tx.callback;
+                callback_param = desc->async_tx.callback_param;
+
+                /* Remove from ld_queue list */
+                list_splice_init(&desc->tx_list, &sh_chan->ld_free);
+
+                dev_dbg(sh_chan->dev, "link descriptor %p will be recycle.\n",
+                                desc);
+
+                list_move(&desc->node, &sh_chan->ld_free);
+                /* Run the link descriptor callback function */
+                if (callback) {
+                        spin_unlock_bh(&sh_chan->desc_lock);
+                        dev_dbg(sh_chan->dev, "link descriptor %p callback\n",
+                                        desc);
+                        callback(callback_param);
+                        spin_lock_bh(&sh_chan->desc_lock);
+                }
+        }
+        spin_unlock_bh(&sh_chan->desc_lock);
+}
+
+static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
+{
+        struct list_head *ld_node;
+        struct sh_dmae_regs hw;
+
+        /* DMA work check */
+        if (dmae_is_idle(sh_chan))
+                return;
+
+        /* Find the first un-transfer desciptor */
+        for (ld_node = sh_chan->ld_queue.next;
+                (ld_node != &sh_chan->ld_queue)
+                        && (to_sh_desc(ld_node)->mark == DESC_COMP);
+                ld_node = ld_node->next)
+                cpu_relax();
+
+        if (ld_node != &sh_chan->ld_queue) {
+                /* Get the ld start address from ld_queue */
+                hw = to_sh_desc(ld_node)->hw;
+                dmae_set_reg(sh_chan, hw);
+                dmae_start(sh_chan);
+        }
+}
+
+static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        sh_chan_xfer_ld_queue(sh_chan);
+}
+
+static enum dma_status sh_dmae_is_complete(struct dma_chan *chan,
+                                        dma_cookie_t cookie,
+                                        dma_cookie_t *done,
+                                        dma_cookie_t *used)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        dma_cookie_t last_used;
+        dma_cookie_t last_complete;
+
+        sh_dmae_chan_ld_cleanup(sh_chan);
+
+        last_used = chan->cookie;
+        last_complete = sh_chan->completed_cookie;
+        if (last_complete == -EBUSY)
+                last_complete = last_used;
+
+        if (done)
+                *done = last_complete;
+
+        if (used)
+                *used = last_used;
+
+        return dma_async_is_complete(cookie, last_complete, last_used);
+}
+
+static irqreturn_t sh_dmae_interrupt(int irq, void *data)
+{
+        irqreturn_t ret = IRQ_NONE;
+        struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+
+        if (chcr & CHCR_TE) {
+                /* DMA stop */
+                dmae_halt(sh_chan);
+
+                ret = IRQ_HANDLED;
+                tasklet_schedule(&sh_chan->tasklet);
+        }
+
+        return ret;
+}
+
+#if defined(CONFIG_CPU_SH4)
+static irqreturn_t sh_dmae_err(int irq, void *data)
+{
+        int err = 0;
+        struct sh_dmae_device *shdev = (struct sh_dmae_device *)data;
+
+        /* IRQ Multi */
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                int cnt = 0;
+                switch (irq) {
+#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ)
+                case DMTE6_IRQ:
+                        cnt++;
+#endif
+                case DMTE0_IRQ:
+                        if (dmaor_read_reg(cnt) & (DMAOR_NMIF | DMAOR_AE)) {
+                                disable_irq(irq);
+                                return IRQ_HANDLED;
+                        }
+                default:
+                        return IRQ_NONE;
+                }
+        } else {
+                /* reset dma controller */
+                err = sh_dmae_rst(0);
+                if (err)
+                        return err;
+                if (shdev->pdata.mode & SHDMA_DMAOR1) {
+                        err = sh_dmae_rst(1);
+                        if (err)
+                                return err;
+                }
+                disable_irq(irq);
+                return IRQ_HANDLED;
+        }
+}
+#endif
+
+static void dmae_do_tasklet(unsigned long data)
+{
+        struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
+        struct sh_desc *desc, *_desc, *cur_desc = NULL;
+        u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
+        list_for_each_entry_safe(desc, _desc,
+                                        &sh_chan->ld_queue, node) {
+                if ((desc->hw.sar + desc->hw.tcr) == sar_buf) {
+                        cur_desc = desc;
+                        break;
+                }
+        }
+
+        if (cur_desc) {
+                switch (cur_desc->async_tx.cookie) {
+                case 0: /* other desc data */
+                        break;
+                case -EBUSY: /* last desc */
+                sh_chan->completed_cookie =
+                                cur_desc->async_tx.cookie;
+                        break;
+                default: /* first desc ( 0 < )*/
+                        sh_chan->completed_cookie =
+                                cur_desc->async_tx.cookie - 1;
+                        break;
+                }
+                cur_desc->mark = DESC_COMP;
+        }
+        /* Next desc */
+        sh_chan_xfer_ld_queue(sh_chan);
+        sh_dmae_chan_ld_cleanup(sh_chan);
+}
+
+static unsigned int get_dmae_irq(unsigned int id)
+{
+        unsigned int irq = 0;
+        if (id < ARRAY_SIZE(dmte_irq_map))
+                irq = dmte_irq_map[id];
+        return irq;
+}
+
+static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id)
+{
+        int err;
+        unsigned int irq = get_dmae_irq(id);
+        unsigned long irqflags = IRQF_DISABLED;
+        struct sh_dmae_chan *new_sh_chan;
+
+        /* alloc channel */
+        new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
+        if (!new_sh_chan) {
+                dev_err(shdev->common.dev, "No free memory for allocating "
+                                "dma channels!\n");
+                return -ENOMEM;
+        }
+
+        new_sh_chan->dev = shdev->common.dev;
+        new_sh_chan->id = id;
+
+        /* Init DMA tasklet */
+        tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet,
+                        (unsigned long)new_sh_chan);
+
+        /* Init the channel */
+        dmae_init(new_sh_chan);
+
+        spin_lock_init(&new_sh_chan->desc_lock);
+
+        /* Init descripter manage list */
+        INIT_LIST_HEAD(&new_sh_chan->ld_queue);
+        INIT_LIST_HEAD(&new_sh_chan->ld_free);
+
+        /* copy struct dma_device */
+        new_sh_chan->common.device = &shdev->common;
+
+        /* Add the channel to DMA device channel list */
+        list_add_tail(&new_sh_chan->common.device_node,
+                        &shdev->common.channels);
+        shdev->common.chancnt++;
+
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                irqflags = IRQF_SHARED;
+#if defined(DMTE6_IRQ)
+                if (irq >= DMTE6_IRQ)
+                        irq = DMTE6_IRQ;
+                else
+#endif
+                        irq = DMTE0_IRQ;
+        }
+
+        snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id),
+                        "sh-dmae%d", new_sh_chan->id);
+
+        /* set up channel irq */
+        err = request_irq(irq, &sh_dmae_interrupt,
+                irqflags, new_sh_chan->dev_id, new_sh_chan);
+        if (err) {
+                dev_err(shdev->common.dev, "DMA channel %d request_irq error "
+                        "with return %d\n", id, err);
+                goto err_no_irq;
+        }
+
+        /* CHCR register control function */
+        new_sh_chan->set_chcr = dmae_set_chcr;
+        /* DMARS register control function */
+        new_sh_chan->set_dmars = dmae_set_dmars;
+
+        shdev->chan[id] = new_sh_chan;
+        return 0;
+
+err_no_irq:
+        /* remove from dmaengine device node */
+        list_del(&new_sh_chan->common.device_node);
+        kfree(new_sh_chan);
+        return err;
+}
+
+static void sh_dmae_chan_remove(struct sh_dmae_device *shdev)
+{
+        int i;
+
+        for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) {
+                if (shdev->chan[i]) {
+                        struct sh_dmae_chan *shchan = shdev->chan[i];
+                        if (!(shdev->pdata.mode & SHDMA_MIX_IRQ))
+                                free_irq(dmte_irq_map[i], shchan);
+
+                        list_del(&shchan->common.device_node);
+                        kfree(shchan);
+                        shdev->chan[i] = NULL;
+                }
+        }
+        shdev->common.chancnt = 0;
+}
+
+static int __init sh_dmae_probe(struct platform_device *pdev)
+{
+        int err = 0, cnt, ecnt;
+        unsigned long irqflags = IRQF_DISABLED;
+#if defined(CONFIG_CPU_SH4)
+        int eirq[] = { DMAE0_IRQ,
+#if defined(DMAE1_IRQ)
+                        DMAE1_IRQ
+#endif
+                };
+#endif
+        struct sh_dmae_device *shdev;
+
+        shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL);
+        if (!shdev) {
+                dev_err(&pdev->dev, "No enough memory\n");
+                err = -ENOMEM;
+                goto shdev_err;
+        }
+
+        /* get platform data */
+        if (!pdev->dev.platform_data)
+                goto shdev_err;
+
+        /* platform data */
+        memcpy(&shdev->pdata, pdev->dev.platform_data,
+                        sizeof(struct sh_dmae_pdata));
+
+        /* reset dma controller */
+        err = sh_dmae_rst(0);
+        if (err)
+                goto rst_err;
+
+        /* SH7780/85/23 has DMAOR1 */
+        if (shdev->pdata.mode & SHDMA_DMAOR1) {
+                err = sh_dmae_rst(1);
+                if (err)
+                        goto rst_err;
+        }
+
+        INIT_LIST_HEAD(&shdev->common.channels);
+
+        dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask);
+        shdev->common.device_alloc_chan_resources
+                = sh_dmae_alloc_chan_resources;
+        shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources;
+        shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy;
+        shdev->common.device_is_tx_complete = sh_dmae_is_complete;
+        shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending;
+        shdev->common.dev = &pdev->dev;
+
+#if defined(CONFIG_CPU_SH4)
+        /* Non Mix IRQ mode SH7722/SH7730 etc... */
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                irqflags = IRQF_SHARED;
+                eirq[0] = DMTE0_IRQ;
+#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ)
+                eirq[1] = DMTE6_IRQ;
+#endif
+        }
+
+        for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) {
+                err = request_irq(eirq[ecnt], sh_dmae_err,
+                        irqflags, "DMAC Address Error", shdev);
+                if (err) {
+                        dev_err(&pdev->dev, "DMA device request_irq"
+                                "error (irq %d) with return %d\n",
+                                eirq[ecnt], err);
+                        goto eirq_err;
+                }
+        }
+#endif /* CONFIG_CPU_SH4 */
+
+        /* Create DMA Channel */
+        for (cnt = 0 ; cnt < MAX_DMA_CHANNELS ; cnt++) {
+                err = sh_dmae_chan_probe(shdev, cnt);
+                if (err)
+                        goto chan_probe_err;
+        }
+
+        platform_set_drvdata(pdev, shdev);
+        dma_async_device_register(&shdev->common);
+
+        return err;
+
+chan_probe_err:
+        sh_dmae_chan_remove(shdev);
+
+eirq_err:
+        for (ecnt-- ; ecnt >= 0; ecnt--)
+                free_irq(eirq[ecnt], shdev);
+
+rst_err:
+        kfree(shdev);
+
+shdev_err:
+        return err;
+}
+
+static int __exit sh_dmae_remove(struct platform_device *pdev)
+{
+        struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
+
+        dma_async_device_unregister(&shdev->common);
+
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                free_irq(DMTE0_IRQ, shdev);
+#if defined(DMTE6_IRQ)
+                free_irq(DMTE6_IRQ, shdev);
+#endif
+        }
+
+        /* channel data remove */
+        sh_dmae_chan_remove(shdev);
+
+        if (!(shdev->pdata.mode & SHDMA_MIX_IRQ)) {
+                free_irq(DMAE0_IRQ, shdev);
+#if defined(DMAE1_IRQ)
+                free_irq(DMAE1_IRQ, shdev);
+#endif
+        }
+        kfree(shdev);
+
+        return 0;
+}
+
+static void sh_dmae_shutdown(struct platform_device *pdev)
+{
+        struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
+        sh_dmae_ctl_stop(0);
+        if (shdev->pdata.mode & SHDMA_DMAOR1)
+                sh_dmae_ctl_stop(1);
+}
+
+static struct platform_driver sh_dmae_driver = {
+        .remove         = __exit_p(sh_dmae_remove),
+        .shutdown       = sh_dmae_shutdown,
+        .driver = {
+                .name   = "sh-dma-engine",
+        },
+};
+
+static int __init sh_dmae_init(void)
+{
+        return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe);
+}
+module_init(sh_dmae_init);
+
+static void __exit sh_dmae_exit(void)
+{
+        platform_driver_unregister(&sh_dmae_driver);
+}
+module_exit(sh_dmae_exit);
+
+MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
+MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/shdma.h b/drivers/dma/shdma.h
new file mode 100644
index 000000000000..2b4bc15a2c0a
--- /dev/null
+++ b/drivers/dma/shdma.h
@@ -0,0 +1,64 @@
+/*
+ * Renesas SuperH DMA Engine support
+ *
+ * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
+ * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#ifndef __DMA_SHDMA_H
+#define __DMA_SHDMA_H
+
+#include <linux/device.h>
+#include <linux/dmapool.h>
+#include <linux/dmaengine.h>
+
+#define SH_DMA_TCR_MAX 0x00FFFFFF       /* 16MB */
+
+struct sh_dmae_regs {
+        u32 sar; /* SAR / source address */
+        u32 dar; /* DAR / destination address */
+        u32 tcr; /* TCR / transfer count */
+};
+
+struct sh_desc {
+        struct list_head tx_list;
+        struct sh_dmae_regs hw;
+        struct list_head node;
+        struct dma_async_tx_descriptor async_tx;
+        int mark;
+};
+
+struct sh_dmae_chan {
+        dma_cookie_t completed_cookie;  /* The maximum cookie completed */
+        spinlock_t desc_lock;                   /* Descriptor operation lock */
+        struct list_head ld_queue;              /* Link descriptors queue */
+        struct list_head ld_free;               /* Link descriptors free */
+        struct dma_chan common;                 /* DMA common channel */
+        struct device *dev;                             /* Channel device */
+        struct tasklet_struct tasklet;  /* Tasklet */
+        int descs_allocated;                    /* desc count */
+        int id;                         /* Raw id of this channel */
+        char dev_id[16];        /* unique name per DMAC of channel */
+
+        /* Set chcr */
+        int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs);
+        /* Set DMA resource */
+        int (*set_dmars)(struct sh_dmae_chan *sh_chan, u16 res);
+};
+
+struct sh_dmae_device {
+        struct dma_device common;
+        struct sh_dmae_chan *chan[MAX_DMA_CHANNELS];
+        struct sh_dmae_pdata pdata;
+};
+
+#define to_sh_chan(chan) container_of(chan, struct sh_dmae_chan, common)
+#define to_sh_desc(lh) container_of(lh, struct sh_desc, node)
+#define tx_to_sh_desc(tx) container_of(tx, struct sh_desc, async_tx)
+
+#endif  /* __DMA_SHDMA_H */
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index 7837930146a4..fb6bb64e8861 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -180,9 +180,8 @@ static struct txx9dmac_desc *txx9dmac_first_queued(struct txx9dmac_chan *dc)
 
 static struct txx9dmac_desc *txx9dmac_last_child(struct txx9dmac_desc *desc)
 {
-        if (!list_empty(&desc->txd.tx_list))
-                desc = list_entry(desc->txd.tx_list.prev,
-                                  struct txx9dmac_desc, desc_node);
+        if (!list_empty(&desc->tx_list))
+                desc = list_entry(desc->tx_list.prev, typeof(*desc), desc_node);
         return desc;
 }
 
@@ -197,6 +196,7 @@ static struct txx9dmac_desc *txx9dmac_desc_alloc(struct txx9dmac_chan *dc,
         desc = kzalloc(sizeof(*desc), flags);
         if (!desc)
                 return NULL;
+        INIT_LIST_HEAD(&desc->tx_list);
         dma_async_tx_descriptor_init(&desc->txd, &dc->chan);
         desc->txd.tx_submit = txx9dmac_tx_submit;
         /* txd.flags will be overwritten in prep funcs */
@@ -245,7 +245,7 @@ static void txx9dmac_sync_desc_for_cpu(struct txx9dmac_chan *dc,
         struct txx9dmac_dev *ddev = dc->ddev;
         struct txx9dmac_desc *child;
 
-        list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &desc->tx_list, desc_node)
                 dma_sync_single_for_cpu(chan2parent(&dc->chan),
                                 child->txd.phys, ddev->descsize,
                                 DMA_TO_DEVICE);
@@ -267,11 +267,11 @@ static void txx9dmac_desc_put(struct txx9dmac_chan *dc,
                 txx9dmac_sync_desc_for_cpu(dc, desc);
 
                 spin_lock_bh(&dc->lock);
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         dev_vdbg(chan2dev(&dc->chan),
                                  "moving child desc %p to freelist\n",
                                  child);
-                list_splice_init(&desc->txd.tx_list, &dc->free_list);
+                list_splice_init(&desc->tx_list, &dc->free_list);
                 dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n",
                          desc);
                 list_add(&desc->desc_node, &dc->free_list);
@@ -429,7 +429,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
         param = txd->callback_param;
 
         txx9dmac_sync_desc_for_cpu(dc, desc);
-        list_splice_init(&txd->tx_list, &dc->free_list);
+        list_splice_init(&desc->tx_list, &dc->free_list);
         list_move(&desc->desc_node, &dc->free_list);
 
         if (!ds) {
@@ -571,7 +571,7 @@ static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr)
                  "Bad descriptor submitted for DMA! (cookie: %d)\n",
                  bad_desc->txd.cookie);
         txx9dmac_dump_desc(dc, &bad_desc->hwdesc);
-        list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                 txx9dmac_dump_desc(dc, &child->hwdesc);
         /* Pretend the descriptor completed successfully */
         txx9dmac_descriptor_complete(dc, bad_desc);
@@ -613,7 +613,7 @@ static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc)
                         return;
                 }
 
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         if (desc_read_CHAR(dc, child) == chain) {
                                 /* Currently in progress */
                                 if (csr & TXX9_DMA_CSR_ABCHC)
@@ -823,8 +823,7 @@ txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                         dma_sync_single_for_device(chan2parent(&dc->chan),
                                         prev->txd.phys, ddev->descsize,
                                         DMA_TO_DEVICE);
-                        list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                        list_add_tail(&desc->desc_node, &first->tx_list);
                 }
                 prev = desc;
         }
@@ -919,8 +918,7 @@ txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                         prev->txd.phys,
                                         ddev->descsize,
                                         DMA_TO_DEVICE);
-                        list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                        list_add_tail(&desc->desc_node, &first->tx_list);
                 }
                 prev = desc;
         }
diff --git a/drivers/dma/txx9dmac.h b/drivers/dma/txx9dmac.h
index c907ff01d276..365d42366b9f 100644
--- a/drivers/dma/txx9dmac.h
+++ b/drivers/dma/txx9dmac.h
@@ -231,6 +231,7 @@ struct txx9dmac_desc {
 
         /* THEN values for driver housekeeping */
         struct list_head                desc_node ____cacheline_aligned;
+        struct list_head                tx_list;
         struct dma_async_tx_descriptor  txd;
         size_t                          len;
 };
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index a3ca18e2d7cf..02127e59fe8e 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -133,6 +133,13 @@ config EDAC_I3000
           Support for error detection and correction on the Intel
           3000 and 3010 server chipsets.
 
+config EDAC_I3200
+        tristate "Intel 3200"
+        depends on EDAC_MM_EDAC && PCI && X86 && EXPERIMENTAL
+        help
+          Support for error detection and correction on the Intel
+          3200 and 3210 server chipsets.
+
 config EDAC_X38
         tristate "Intel X38"
         depends on EDAC_MM_EDAC && PCI && X86
@@ -176,11 +183,11 @@ config EDAC_I5100
           San Clemente MCH.
 
 config EDAC_MPC85XX
-        tristate "Freescale MPC85xx"
-        depends on EDAC_MM_EDAC && FSL_SOC && MPC85xx
+        tristate "Freescale MPC83xx / MPC85xx"
+        depends on EDAC_MM_EDAC && FSL_SOC && (PPC_83xx || MPC85xx)
         help
           Support for error detection and correction on the Freescale
-          MPC8560, MPC8540, MPC8548
+          MPC8349, MPC8560, MPC8540, MPC8548
 
 config EDAC_MV64X60
         tristate "Marvell MV64x60"
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index cfa033ce53a7..7a473bbe8abd 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -32,6 +32,7 @@ obj-$(CONFIG_EDAC_I82443BXGX)		+= i82443bxgx_edac.o
 obj-$(CONFIG_EDAC_I82875P)              += i82875p_edac.o
 obj-$(CONFIG_EDAC_I82975X)              += i82975x_edac.o
 obj-$(CONFIG_EDAC_I3000)                += i3000_edac.o
+obj-$(CONFIG_EDAC_I3200)                += i3200_edac.o
 obj-$(CONFIG_EDAC_X38)                  += x38_edac.o
 obj-$(CONFIG_EDAC_I82860)               += i82860_edac.o
 obj-$(CONFIG_EDAC_R82600)               += r82600_edac.o
@@ -49,3 +50,4 @@ obj-$(CONFIG_EDAC_CELL)			+= cell_edac.o
 obj-$(CONFIG_EDAC_PPC4XX)               += ppc4xx_edac.o
 obj-$(CONFIG_EDAC_AMD8111)              += amd8111_edac.o
 obj-$(CONFIG_EDAC_AMD8131)              += amd8131_edac.o
+
diff --git a/drivers/edac/cpc925_edac.c b/drivers/edac/cpc925_edac.c
index 8c54196b5aba..3d50274f1348 100644
--- a/drivers/edac/cpc925_edac.c
+++ b/drivers/edac/cpc925_edac.c
@@ -885,14 +885,14 @@ static int __devinit cpc925_probe(struct platform_device *pdev)
 
         if (!devm_request_mem_region(&pdev->dev,
                                      r->start,
-                                     r->end - r->start + 1,
+                                     resource_size(r),
                                      pdev->name)) {
                 cpc925_printk(KERN_ERR, "Unable to request mem region\n");
                 res = -EBUSY;
                 goto err1;
         }
 
-        vbase = devm_ioremap(&pdev->dev, r->start, r->end - r->start + 1);
+        vbase = devm_ioremap(&pdev->dev, r->start, resource_size(r));
         if (!vbase) {
                 cpc925_printk(KERN_ERR, "Unable to ioremap device\n");
                 res = -ENOMEM;
@@ -953,7 +953,7 @@ err3:
         cpc925_mc_exit(mci);
         edac_mc_free(mci);
 err2:
-        devm_release_mem_region(&pdev->dev, r->start, r->end-r->start+1);
+        devm_release_mem_region(&pdev->dev, r->start, resource_size(r));
 err1:
         devres_release_group(&pdev->dev, cpc925_probe);
 out:
diff --git a/drivers/edac/edac_device.c b/drivers/edac/edac_device.c
index b02a6a69a8f0..d5e13c94714f 100644
--- a/drivers/edac/edac_device.c
+++ b/drivers/edac/edac_device.c
@@ -356,7 +356,6 @@ static void complete_edac_device_list_del(struct rcu_head *head)
 
         edac_dev = container_of(head, struct edac_device_ctl_info, rcu);
         INIT_LIST_HEAD(&edac_dev->link);
-        complete(&edac_dev->removal_complete);
 }
 
 /*
@@ -369,10 +368,8 @@ static void del_edac_device_from_global_list(struct edac_device_ctl_info
                                                 *edac_device)
 {
         list_del_rcu(&edac_device->link);
-
-        init_completion(&edac_device->removal_complete);
         call_rcu(&edac_device->rcu, complete_edac_device_list_del);
-        wait_for_completion(&edac_device->removal_complete);
+        rcu_barrier();
 }
 
 /*
diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c
index 335b7ebdb11c..b629c41756f0 100644
--- a/drivers/edac/edac_mc.c
+++ b/drivers/edac/edac_mc.c
@@ -418,16 +418,14 @@ static void complete_mc_list_del(struct rcu_head *head)
 
         mci = container_of(head, struct mem_ctl_info, rcu);
         INIT_LIST_HEAD(&mci->link);
-        complete(&mci->complete);
 }
 
 static void del_mc_from_global_list(struct mem_ctl_info *mci)
 {
         atomic_dec(&edac_handlers);
         list_del_rcu(&mci->link);
-        init_completion(&mci->complete);
         call_rcu(&mci->rcu, complete_mc_list_del);
-        wait_for_completion(&mci->complete);
+        rcu_barrier();
 }
 
 /**
diff --git a/drivers/edac/edac_pci.c b/drivers/edac/edac_pci.c
index 30b585b1d60b..efb5d5650783 100644
--- a/drivers/edac/edac_pci.c
+++ b/drivers/edac/edac_pci.c
@@ -174,7 +174,6 @@ static void complete_edac_pci_list_del(struct rcu_head *head)
 
         pci = container_of(head, struct edac_pci_ctl_info, rcu);
         INIT_LIST_HEAD(&pci->link);
-        complete(&pci->complete);
 }
 
 /*
@@ -185,9 +184,8 @@ static void complete_edac_pci_list_del(struct rcu_head *head)
 static void del_edac_pci_from_global_list(struct edac_pci_ctl_info *pci)
 {
         list_del_rcu(&pci->link);
-        init_completion(&pci->complete);
         call_rcu(&pci->rcu, complete_edac_pci_list_del);
-        wait_for_completion(&pci->complete);
+        rcu_barrier();
 }
 
 #if 0
diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c
new file mode 100644
index 000000000000..fde4db91c4d2
--- /dev/null
+++ b/drivers/edac/i3200_edac.c
@@ -0,0 +1,527 @@
+/*
+ * Intel 3200/3210 Memory Controller kernel module
+ * Copyright (C) 2008-2009 Akamai Technologies, Inc.
+ * Portions by Hitoshi Mitake <h.mitake@gmail.com>.
+ *
+ * This file may be distributed under the terms of the
+ * GNU General Public License.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/slab.h>
+#include <linux/edac.h>
+#include <linux/io.h>
+#include "edac_core.h"
+
+#define I3200_REVISION        "1.1"
+
+#define EDAC_MOD_STR        "i3200_edac"
+
+#define PCI_DEVICE_ID_INTEL_3200_HB    0x29f0
+
+#define I3200_RANKS             8
+#define I3200_RANKS_PER_CHANNEL 4
+#define I3200_CHANNELS          2
+
+/* Intel 3200 register addresses - device 0 function 0 - DRAM Controller */
+
+#define I3200_MCHBAR_LOW        0x48    /* MCH Memory Mapped Register BAR */
+#define I3200_MCHBAR_HIGH       0x4c
+#define I3200_MCHBAR_MASK       0xfffffc000ULL  /* bits 35:14 */
+#define I3200_MMR_WINDOW_SIZE   16384
+
+#define I3200_TOM               0xa0    /* Top of Memory (16b)
+                 *
+                 * 15:10 reserved
+                 *  9:0  total populated physical memory
+                 */
+#define I3200_TOM_MASK          0x3ff   /* bits 9:0 */
+#define I3200_TOM_SHIFT         26      /* 64MiB grain */
+
+#define I3200_ERRSTS            0xc8    /* Error Status Register (16b)
+                 *
+                 * 15    reserved
+                 * 14    Isochronous TBWRR Run Behind FIFO Full
+                 *       (ITCV)
+                 * 13    Isochronous TBWRR Run Behind FIFO Put
+                 *       (ITSTV)
+                 * 12    reserved
+                 * 11    MCH Thermal Sensor Event
+                 *       for SMI/SCI/SERR (GTSE)
+                 * 10    reserved
+                 *  9    LOCK to non-DRAM Memory Flag (LCKF)
+                 *  8    reserved
+                 *  7    DRAM Throttle Flag (DTF)
+                 *  6:2  reserved
+                 *  1    Multi-bit DRAM ECC Error Flag (DMERR)
+                 *  0    Single-bit DRAM ECC Error Flag (DSERR)
+                 */
+#define I3200_ERRSTS_UE         0x0002
+#define I3200_ERRSTS_CE         0x0001
+#define I3200_ERRSTS_BITS       (I3200_ERRSTS_UE | I3200_ERRSTS_CE)
+
+
+/* Intel  MMIO register space - device 0 function 0 - MMR space */
+
+#define I3200_C0DRB     0x200   /* Channel 0 DRAM Rank Boundary (16b x 4)
+                 *
+                 * 15:10 reserved
+                 *  9:0  Channel 0 DRAM Rank Boundary Address
+                 */
+#define I3200_C1DRB     0x600   /* Channel 1 DRAM Rank Boundary (16b x 4) */
+#define I3200_DRB_MASK  0x3ff   /* bits 9:0 */
+#define I3200_DRB_SHIFT 26      /* 64MiB grain */
+
+#define I3200_C0ECCERRLOG       0x280   /* Channel 0 ECC Error Log (64b)
+                 *
+                 * 63:48 Error Column Address (ERRCOL)
+                 * 47:32 Error Row Address (ERRROW)
+                 * 31:29 Error Bank Address (ERRBANK)
+                 * 28:27 Error Rank Address (ERRRANK)
+                 * 26:24 reserved
+                 * 23:16 Error Syndrome (ERRSYND)
+                 * 15: 2 reserved
+                 *    1  Multiple Bit Error Status (MERRSTS)
+                 *    0  Correctable Error Status (CERRSTS)
+                 */
+#define I3200_C1ECCERRLOG               0x680   /* Chan 1 ECC Error Log (64b) */
+#define I3200_ECCERRLOG_CE              0x1
+#define I3200_ECCERRLOG_UE              0x2
+#define I3200_ECCERRLOG_RANK_BITS       0x18000000
+#define I3200_ECCERRLOG_RANK_SHIFT      27
+#define I3200_ECCERRLOG_SYNDROME_BITS   0xff0000
+#define I3200_ECCERRLOG_SYNDROME_SHIFT  16
+#define I3200_CAPID0                    0xe0    /* P.95 of spec for details */
+
+struct i3200_priv {
+        void __iomem *window;
+};
+
+static int nr_channels;
+
+static int how_many_channels(struct pci_dev *pdev)
+{
+        unsigned char capid0_8b; /* 8th byte of CAPID0 */
+
+        pci_read_config_byte(pdev, I3200_CAPID0 + 8, &capid0_8b);
+        if (capid0_8b & 0x20) { /* check DCD: Dual Channel Disable */
+                debugf0("In single channel mode.\n");
+                return 1;
+        } else {
+                debugf0("In dual channel mode.\n");
+                return 2;
+        }
+}
+
+static unsigned long eccerrlog_syndrome(u64 log)
+{
+        return (log & I3200_ECCERRLOG_SYNDROME_BITS) >>
+                I3200_ECCERRLOG_SYNDROME_SHIFT;
+}
+
+static int eccerrlog_row(int channel, u64 log)
+{
+        u64 rank = ((log & I3200_ECCERRLOG_RANK_BITS) >>
+                I3200_ECCERRLOG_RANK_SHIFT);
+        return rank | (channel * I3200_RANKS_PER_CHANNEL);
+}
+
+enum i3200_chips {
+        I3200 = 0,
+};
+
+struct i3200_dev_info {
+        const char *ctl_name;
+};
+
+struct i3200_error_info {
+        u16 errsts;
+        u16 errsts2;
+        u64 eccerrlog[I3200_CHANNELS];
+};
+
+static const struct i3200_dev_info i3200_devs[] = {
+        [I3200] = {
+                .ctl_name = "i3200"
+        },
+};
+
+static struct pci_dev *mci_pdev;
+static int i3200_registered = 1;
+
+
+static void i3200_clear_error_info(struct mem_ctl_info *mci)
+{
+        struct pci_dev *pdev;
+
+        pdev = to_pci_dev(mci->dev);
+
+        /*
+         * Clear any error bits.
+         * (Yes, we really clear bits by writing 1 to them.)
+         */
+        pci_write_bits16(pdev, I3200_ERRSTS, I3200_ERRSTS_BITS,
+                I3200_ERRSTS_BITS);
+}
+
+static void i3200_get_and_clear_error_info(struct mem_ctl_info *mci,
+                struct i3200_error_info *info)
+{
+        struct pci_dev *pdev;
+        struct i3200_priv *priv = mci->pvt_info;
+        void __iomem *window = priv->window;
+
+        pdev = to_pci_dev(mci->dev);
+
+        /*
+         * This is a mess because there is no atomic way to read all the
+         * registers at once and the registers can transition from CE being
+         * overwritten by UE.
+         */
+        pci_read_config_word(pdev, I3200_ERRSTS, &info->errsts);
+        if (!(info->errsts & I3200_ERRSTS_BITS))
+                return;
+
+        info->eccerrlog[0] = readq(window + I3200_C0ECCERRLOG);
+        if (nr_channels == 2)
+                info->eccerrlog[1] = readq(window + I3200_C1ECCERRLOG);
+
+        pci_read_config_word(pdev, I3200_ERRSTS, &info->errsts2);
+
+        /*
+         * If the error is the same for both reads then the first set
+         * of reads is valid.  If there is a change then there is a CE
+         * with no info and the second set of reads is valid and
+         * should be UE info.
+         */
+        if ((info->errsts ^ info->errsts2) & I3200_ERRSTS_BITS) {
+                info->eccerrlog[0] = readq(window + I3200_C0ECCERRLOG);
+                if (nr_channels == 2)
+                        info->eccerrlog[1] = readq(window + I3200_C1ECCERRLOG);
+        }
+
+        i3200_clear_error_info(mci);
+}
+
+static void i3200_process_error_info(struct mem_ctl_info *mci,
+                struct i3200_error_info *info)
+{
+        int channel;
+        u64 log;
+
+        if (!(info->errsts & I3200_ERRSTS_BITS))
+                return;
+
+        if ((info->errsts ^ info->errsts2) & I3200_ERRSTS_BITS) {
+                edac_mc_handle_ce_no_info(mci, "UE overwrote CE");
+                info->errsts = info->errsts2;
+        }
+
+        for (channel = 0; channel < nr_channels; channel++) {
+                log = info->eccerrlog[channel];
+                if (log & I3200_ECCERRLOG_UE) {
+                        edac_mc_handle_ue(mci, 0, 0,
+                                eccerrlog_row(channel, log),
+                                "i3200 UE");
+                } else if (log & I3200_ECCERRLOG_CE) {
+                        edac_mc_handle_ce(mci, 0, 0,
+                                eccerrlog_syndrome(log),
+                                eccerrlog_row(channel, log), 0,
+                                "i3200 CE");
+                }
+        }
+}
+
+static void i3200_check(struct mem_ctl_info *mci)
+{
+        struct i3200_error_info info;
+
+        debugf1("MC%d: %s()\n", mci->mc_idx, __func__);
+        i3200_get_and_clear_error_info(mci, &info);
+        i3200_process_error_info(mci, &info);
+}
+
+
+void __iomem *i3200_map_mchbar(struct pci_dev *pdev)
+{
+        union {
+                u64 mchbar;
+                struct {
+                        u32 mchbar_low;
+                        u32 mchbar_high;
+                };
+        } u;
+        void __iomem *window;
+
+        pci_read_config_dword(pdev, I3200_MCHBAR_LOW, &u.mchbar_low);
+        pci_read_config_dword(pdev, I3200_MCHBAR_HIGH, &u.mchbar_high);
+        u.mchbar &= I3200_MCHBAR_MASK;
+
+        if (u.mchbar != (resource_size_t)u.mchbar) {
+                printk(KERN_ERR
+                        "i3200: mmio space beyond accessible range (0x%llx)\n",
+                        (unsigned long long)u.mchbar);
+                return NULL;
+        }
+
+        window = ioremap_nocache(u.mchbar, I3200_MMR_WINDOW_SIZE);
+        if (!window)
+                printk(KERN_ERR "i3200: cannot map mmio space at 0x%llx\n",
+                        (unsigned long long)u.mchbar);
+
+        return window;
+}
+
+
+static void i3200_get_drbs(void __iomem *window,
+        u16 drbs[I3200_CHANNELS][I3200_RANKS_PER_CHANNEL])
+{
+        int i;
+
+        for (i = 0; i < I3200_RANKS_PER_CHANNEL; i++) {
+                drbs[0][i] = readw(window + I3200_C0DRB + 2*i) & I3200_DRB_MASK;
+                drbs[1][i] = readw(window + I3200_C1DRB + 2*i) & I3200_DRB_MASK;
+        }
+}
+
+static bool i3200_is_stacked(struct pci_dev *pdev,
+        u16 drbs[I3200_CHANNELS][I3200_RANKS_PER_CHANNEL])
+{
+        u16 tom;
+
+        pci_read_config_word(pdev, I3200_TOM, &tom);
+        tom &= I3200_TOM_MASK;
+
+        return drbs[I3200_CHANNELS - 1][I3200_RANKS_PER_CHANNEL - 1] == tom;
+}
+
+static unsigned long drb_to_nr_pages(
+        u16 drbs[I3200_CHANNELS][I3200_RANKS_PER_CHANNEL], bool stacked,
+        int channel, int rank)
+{
+        int n;
+
+        n = drbs[channel][rank];
+        if (rank > 0)
+                n -= drbs[channel][rank - 1];
+        if (stacked && (channel == 1) &&
+        drbs[channel][rank] == drbs[channel][I3200_RANKS_PER_CHANNEL - 1])
+                n -= drbs[0][I3200_RANKS_PER_CHANNEL - 1];
+
+        n <<= (I3200_DRB_SHIFT - PAGE_SHIFT);
+        return n;
+}
+
+static int i3200_probe1(struct pci_dev *pdev, int dev_idx)
+{
+        int rc;
+        int i;
+        struct mem_ctl_info *mci = NULL;
+        unsigned long last_page;
+        u16 drbs[I3200_CHANNELS][I3200_RANKS_PER_CHANNEL];
+        bool stacked;
+        void __iomem *window;
+        struct i3200_priv *priv;
+
+        debugf0("MC: %s()\n", __func__);
+
+        window = i3200_map_mchbar(pdev);
+        if (!window)
+                return -ENODEV;
+
+        i3200_get_drbs(window, drbs);
+        nr_channels = how_many_channels(pdev);
+
+        mci = edac_mc_alloc(sizeof(struct i3200_priv), I3200_RANKS,
+                nr_channels, 0);
+        if (!mci)
+                return -ENOMEM;
+
+        debugf3("MC: %s(): init mci\n", __func__);
+
+        mci->dev = &pdev->dev;
+        mci->mtype_cap = MEM_FLAG_DDR2;
+
+        mci->edac_ctl_cap = EDAC_FLAG_SECDED;
+        mci->edac_cap = EDAC_FLAG_SECDED;
+
+        mci->mod_name = EDAC_MOD_STR;
+        mci->mod_ver = I3200_REVISION;
+        mci->ctl_name = i3200_devs[dev_idx].ctl_name;
+        mci->dev_name = pci_name(pdev);
+        mci->edac_check = i3200_check;
+        mci->ctl_page_to_phys = NULL;
+        priv = mci->pvt_info;
+        priv->window = window;
+
+        stacked = i3200_is_stacked(pdev, drbs);
+
+        /*
+         * The dram rank boundary (DRB) reg values are boundary addresses
+         * for each DRAM rank with a granularity of 64MB.  DRB regs are
+         * cumulative; the last one will contain the total memory
+         * contained in all ranks.
+         */
+        last_page = -1UL;
+        for (i = 0; i < mci->nr_csrows; i++) {
+                unsigned long nr_pages;
+                struct csrow_info *csrow = &mci->csrows[i];
+
+                nr_pages = drb_to_nr_pages(drbs, stacked,
+                        i / I3200_RANKS_PER_CHANNEL,
+                        i % I3200_RANKS_PER_CHANNEL);
+
+                if (nr_pages == 0) {
+                        csrow->mtype = MEM_EMPTY;
+                        continue;
+                }
+
+                csrow->first_page = last_page + 1;
+                last_page += nr_pages;
+                csrow->last_page = last_page;
+                csrow->nr_pages = nr_pages;
+
+                csrow->grain = nr_pages << PAGE_SHIFT;
+                csrow->mtype = MEM_DDR2;
+                csrow->dtype = DEV_UNKNOWN;
+                csrow->edac_mode = EDAC_UNKNOWN;
+        }
+
+        i3200_clear_error_info(mci);
+
+        rc = -ENODEV;
+        if (edac_mc_add_mc(mci)) {
+                debugf3("MC: %s(): failed edac_mc_add_mc()\n", __func__);
+                goto fail;
+        }
+
+        /* get this far and it's successful */
+        debugf3("MC: %s(): success\n", __func__);
+        return 0;
+
+fail:
+        iounmap(window);
+        if (mci)
+                edac_mc_free(mci);
+
+        return rc;
+}
+
+static int __devinit i3200_init_one(struct pci_dev *pdev,
+                const struct pci_device_id *ent)
+{
+        int rc;
+
+        debugf0("MC: %s()\n", __func__);
+
+        if (pci_enable_device(pdev) < 0)
+                return -EIO;
+
+        rc = i3200_probe1(pdev, ent->driver_data);
+        if (!mci_pdev)
+                mci_pdev = pci_dev_get(pdev);
+
+        return rc;
+}
+
+static void __devexit i3200_remove_one(struct pci_dev *pdev)
+{
+        struct mem_ctl_info *mci;
+        struct i3200_priv *priv;
+
+        debugf0("%s()\n", __func__);
+
+        mci = edac_mc_del_mc(&pdev->dev);
+        if (!mci)
+                return;
+
+        priv = mci->pvt_info;
+        iounmap(priv->window);
+
+        edac_mc_free(mci);
+}
+
+static const struct pci_device_id i3200_pci_tbl[] __devinitdata = {
+        {
+                PCI_VEND_DEV(INTEL, 3200_HB), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
+                I3200},
+        {
+                0,
+        }            /* 0 terminated list. */
+};
+
+MODULE_DEVICE_TABLE(pci, i3200_pci_tbl);
+
+static struct pci_driver i3200_driver = {
+        .name = EDAC_MOD_STR,
+        .probe = i3200_init_one,
+        .remove = __devexit_p(i3200_remove_one),
+        .id_table = i3200_pci_tbl,
+};
+
+static int __init i3200_init(void)
+{
+        int pci_rc;
+
+        debugf3("MC: %s()\n", __func__);
+
+        /* Ensure that the OPSTATE is set correctly for POLL or NMI */
+        opstate_init();
+
+        pci_rc = pci_register_driver(&i3200_driver);
+        if (pci_rc < 0)
+                goto fail0;
+
+        if (!mci_pdev) {
+                i3200_registered = 0;
+                mci_pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
+                                PCI_DEVICE_ID_INTEL_3200_HB, NULL);
+                if (!mci_pdev) {
+                        debugf0("i3200 pci_get_device fail\n");
+                        pci_rc = -ENODEV;
+                        goto fail1;
+                }
+
+                pci_rc = i3200_init_one(mci_pdev, i3200_pci_tbl);
+                if (pci_rc < 0) {
+                        debugf0("i3200 init fail\n");
+                        pci_rc = -ENODEV;
+                        goto fail1;
+                }
+        }
+
+        return 0;
+
+fail1:
+        pci_unregister_driver(&i3200_driver);
+
+fail0:
+        if (mci_pdev)
+                pci_dev_put(mci_pdev);
+
+        return pci_rc;
+}
+
+static void __exit i3200_exit(void)
+{
+        debugf3("MC: %s()\n", __func__);
+
+        pci_unregister_driver(&i3200_driver);
+        if (!i3200_registered) {
+                i3200_remove_one(mci_pdev);
+                pci_dev_put(mci_pdev);
+        }
+}
+
+module_init(i3200_init);
+module_exit(i3200_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Akamai Technologies, Inc.");
+MODULE_DESCRIPTION("MC support for Intel 3200 memory hub controllers");
+
+module_param(edac_op_state, int, 0444);
+MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c
index 3f2ccfc6407c..157f6504f25e 100644
--- a/drivers/edac/mpc85xx_edac.c
+++ b/drivers/edac/mpc85xx_edac.c
@@ -41,7 +41,9 @@ static u32 orig_pci_err_en;
 #endif
 
 static u32 orig_l2_err_disable;
+#ifdef CONFIG_MPC85xx
 static u32 orig_hid1[2];
+#endif
 
 /************************ MC SYSFS parts ***********************************/
 
@@ -646,6 +648,7 @@ static struct of_device_id mpc85xx_l2_err_of_match[] = {
         { .compatible = "fsl,mpc8560-l2-cache-controller", },
         { .compatible = "fsl,mpc8568-l2-cache-controller", },
         { .compatible = "fsl,mpc8572-l2-cache-controller", },
+        { .compatible = "fsl,p2020-l2-cache-controller", },
         {},
 };
 
@@ -788,19 +791,20 @@ static void __devinit mpc85xx_init_csrows(struct mem_ctl_info *mci)
                 csrow = &mci->csrows[index];
                 cs_bnds = in_be32(pdata->mc_vbase + MPC85XX_MC_CS_BNDS_0 +
                                   (index * MPC85XX_MC_CS_BNDS_OFS));
-                start = (cs_bnds & 0xfff0000) << 4;
-                end = ((cs_bnds & 0xfff) << 20);
-                if (start)
-                        start |= 0xfffff;
-                if (end)
-                        end |= 0xfffff;
+
+                start = (cs_bnds & 0xffff0000) >> 16;
+                end   = (cs_bnds & 0x0000ffff);
 
                 if (start == end)
                         continue;       /* not populated */
 
+                start <<= (24 - PAGE_SHIFT);
+                end   <<= (24 - PAGE_SHIFT);
+                end    |= (1 << (24 - PAGE_SHIFT)) - 1;
+
                 csrow->first_page = start >> PAGE_SHIFT;
                 csrow->last_page = end >> PAGE_SHIFT;
-                csrow->nr_pages = csrow->last_page + 1 - csrow->first_page;
+                csrow->nr_pages = end + 1 - start;
                 csrow->grain = 8;
                 csrow->mtype = mtype;
                 csrow->dtype = DEV_UNKNOWN;
@@ -984,6 +988,8 @@ static struct of_device_id mpc85xx_mc_err_of_match[] = {
         { .compatible = "fsl,mpc8560-memory-controller", },
         { .compatible = "fsl,mpc8568-memory-controller", },
         { .compatible = "fsl,mpc8572-memory-controller", },
+        { .compatible = "fsl,mpc8349-memory-controller", },
+        { .compatible = "fsl,p2020-memory-controller", },
         {},
 };
 
@@ -999,13 +1005,13 @@ static struct of_platform_driver mpc85xx_mc_err_driver = {
                    },
 };
 
-
+#ifdef CONFIG_MPC85xx
 static void __init mpc85xx_mc_clear_rfxe(void *data)
 {
         orig_hid1[smp_processor_id()] = mfspr(SPRN_HID1);
         mtspr(SPRN_HID1, (orig_hid1[smp_processor_id()] & ~0x20000));
 }
-
+#endif
 
 static int __init mpc85xx_mc_init(void)
 {
@@ -1038,26 +1044,32 @@ static int __init mpc85xx_mc_init(void)
                 printk(KERN_WARNING EDAC_MOD_STR "PCI fails to register\n");
 #endif
 
+#ifdef CONFIG_MPC85xx
         /*
          * need to clear HID1[RFXE] to disable machine check int
          * so we can catch it
          */
         if (edac_op_state == EDAC_OPSTATE_INT)
                 on_each_cpu(mpc85xx_mc_clear_rfxe, NULL, 0);
+#endif
 
         return 0;
 }
 
 module_init(mpc85xx_mc_init);
 
+#ifdef CONFIG_MPC85xx
 static void __exit mpc85xx_mc_restore_hid1(void *data)
 {
         mtspr(SPRN_HID1, orig_hid1[smp_processor_id()]);
 }
+#endif
 
 static void __exit mpc85xx_mc_exit(void)
 {
+#ifdef CONFIG_MPC85xx
         on_each_cpu(mpc85xx_mc_restore_hid1, NULL, 0);
+#endif
 #ifdef CONFIG_PCI
         of_unregister_platform_driver(&mpc85xx_pci_err_driver);
 #endif
diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c
index 5131aaae8e03..a6b9fec13a74 100644
--- a/drivers/edac/mv64x60_edac.c
+++ b/drivers/edac/mv64x60_edac.c
@@ -90,7 +90,7 @@ static int __init mv64x60_pci_fixup(struct platform_device *pdev)
                 return -ENOENT;
         }
 
-        pci_serr = ioremap(r->start, r->end - r->start + 1);
+        pci_serr = ioremap(r->start, resource_size(r));
         if (!pci_serr)
                 return -ENOMEM;
 
@@ -140,7 +140,7 @@ static int __devinit mv64x60_pci_err_probe(struct platform_device *pdev)
 
         if (!devm_request_mem_region(&pdev->dev,
                                      r->start,
-                                     r->end - r->start + 1,
+                                     resource_size(r),
                                      pdata->name)) {
                 printk(KERN_ERR "%s: Error while requesting mem region\n",
                        __func__);
@@ -150,7 +150,7 @@ static int __devinit mv64x60_pci_err_probe(struct platform_device *pdev)
 
         pdata->pci_vbase = devm_ioremap(&pdev->dev,
                                         r->start,
-                                        r->end - r->start + 1);
+                                        resource_size(r));
         if (!pdata->pci_vbase) {
                 printk(KERN_ERR "%s: Unable to setup PCI err regs\n", __func__);
                 res = -ENOMEM;
@@ -306,7 +306,7 @@ static int __devinit mv64x60_sram_err_probe(struct platform_device *pdev)
 
         if (!devm_request_mem_region(&pdev->dev,
                                      r->start,
-                                     r->end - r->start + 1,
+                                     resource_size(r),
                                      pdata->name)) {
                 printk(KERN_ERR "%s: Error while request mem region\n",
                        __func__);
@@ -316,7 +316,7 @@ static int __devinit mv64x60_sram_err_probe(struct platform_device *pdev)
 
         pdata->sram_vbase = devm_ioremap(&pdev->dev,
                                          r->start,
-                                         r->end - r->start + 1);
+                                         resource_size(r));
         if (!pdata->sram_vbase) {
                 printk(KERN_ERR "%s: Unable to setup SRAM err regs\n",
                        __func__);
@@ -474,7 +474,7 @@ static int __devinit mv64x60_cpu_err_probe(struct platform_device *pdev)
 
         if (!devm_request_mem_region(&pdev->dev,
                                      r->start,
-                                     r->end - r->start + 1,
+                                     resource_size(r),
                                      pdata->name)) {
                 printk(KERN_ERR "%s: Error while requesting mem region\n",
                        __func__);
@@ -484,7 +484,7 @@ static int __devinit mv64x60_cpu_err_probe(struct platform_device *pdev)
 
         pdata->cpu_vbase[0] = devm_ioremap(&pdev->dev,
                                            r->start,
-                                           r->end - r->start + 1);
+                                           resource_size(r));
         if (!pdata->cpu_vbase[0]) {
                 printk(KERN_ERR "%s: Unable to setup CPU err regs\n", __func__);
                 res = -ENOMEM;
@@ -501,7 +501,7 @@ static int __devinit mv64x60_cpu_err_probe(struct platform_device *pdev)
 
         if (!devm_request_mem_region(&pdev->dev,
                                      r->start,
-                                     r->end - r->start + 1,
+                                     resource_size(r),
                                      pdata->name)) {
                 printk(KERN_ERR "%s: Error while requesting mem region\n",
                        __func__);
@@ -511,7 +511,7 @@ static int __devinit mv64x60_cpu_err_probe(struct platform_device *pdev)
 
         pdata->cpu_vbase[1] = devm_ioremap(&pdev->dev,
                                            r->start,
-                                           r->end - r->start + 1);
+                                           resource_size(r));
         if (!pdata->cpu_vbase[1]) {
                 printk(KERN_ERR "%s: Unable to setup CPU err regs\n", __func__);
                 res = -ENOMEM;
@@ -726,7 +726,7 @@ static int __devinit mv64x60_mc_err_probe(struct platform_device *pdev)
 
         if (!devm_request_mem_region(&pdev->dev,
                                      r->start,
-                                     r->end - r->start + 1,
+                                     resource_size(r),
                                      pdata->name)) {
                 printk(KERN_ERR "%s: Error while requesting mem region\n",
                        __func__);
@@ -736,7 +736,7 @@ static int __devinit mv64x60_mc_err_probe(struct platform_device *pdev)
 
         pdata->mc_vbase = devm_ioremap(&pdev->dev,
                                        r->start,
-                                       r->end - r->start + 1);
+                                       resource_size(r));
         if (!pdata->mc_vbase) {
                 printk(KERN_ERR "%s: Unable to setup MC err regs\n", __func__);
                 res = -ENOMEM;
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index e4d971c8b9d0..f831ea159291 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -102,6 +102,7 @@ config DRM_I915
         select BACKLIGHT_CLASS_DEVICE if ACPI
         select INPUT if ACPI
         select ACPI_VIDEO if ACPI
+        select ACPI_BUTTON if ACPI
         help
           Choose this option if you have a system that has Intel 830M, 845G,
           852GM, 855GM 865G or 915G integrated graphics.  If M is selected, the
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 230c9ffdd5e9..80391995bdec 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -142,6 +142,19 @@ drm_gem_object_alloc(struct drm_device *dev, size_t size)
         if (IS_ERR(obj->filp))
                 goto free;
 
+        /* Basically we want to disable the OOM killer and handle ENOMEM
+         * ourselves by sacrificing pages from cached buffers.
+         * XXX shmem_file_[gs]et_gfp_mask()
+         */
+        mapping_set_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping,
+                             GFP_HIGHUSER |
+                             __GFP_COLD |
+                             __GFP_FS |
+                             __GFP_RECLAIMABLE |
+                             __GFP_NORETRY |
+                             __GFP_NOWARN |
+                             __GFP_NOMEMALLOC);
+
         kref_init(&obj->refcount);
         kref_init(&obj->handlecount);
         obj->size = size;
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 5269dfa5f620..fa7b9be096bc 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -9,6 +9,7 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
           i915_gem.o \
           i915_gem_debug.o \
           i915_gem_tiling.o \
+          i915_trace_points.o \
           intel_display.o \
           intel_crt.o \
           intel_lvds.o \
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 1e3bdcee863c..f8ce9a3a420d 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -96,11 +96,13 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
         {
                 struct drm_gem_object *obj = obj_priv->obj;
 
-                seq_printf(m, "    %p: %s %08x %08x %d",
+                seq_printf(m, "    %p: %s %8zd %08x %08x %d %s",
                            obj,
                            get_pin_flag(obj_priv),
+                           obj->size,
                            obj->read_domains, obj->write_domain,
-                           obj_priv->last_rendering_seqno);
+                           obj_priv->last_rendering_seqno,
+                           obj_priv->dirty ? "dirty" : "");
 
                 if (obj->name)
                         seq_printf(m, " (name: %d)", obj->name);
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 5a49a1867b35..45d507ebd3ff 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -33,6 +33,7 @@
 #include "intel_drv.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
+#include "i915_trace.h"
 #include <linux/vgaarb.h>
 
 /* Really want an OS-independent resettable timer.  Would like to have
@@ -50,14 +51,18 @@ int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
         u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
         int i;
 
+        trace_i915_ring_wait_begin (dev);
+
         for (i = 0; i < 100000; i++) {
                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
                 acthd = I915_READ(acthd_reg);
                 ring->space = ring->head - (ring->tail + 8);
                 if (ring->space < 0)
                         ring->space += ring->Size;
-                if (ring->space >= n)
+                if (ring->space >= n) {
+                        trace_i915_ring_wait_end (dev);
                         return 0;
+                }
 
                 if (dev->primary->master) {
                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
@@ -77,6 +82,7 @@ int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
 
         }
 
+        trace_i915_ring_wait_end (dev);
         return -EBUSY;
 }
 
@@ -922,7 +928,8 @@ static int i915_get_bridge_dev(struct drm_device *dev)
  * how much was set aside so we can use it for our own purposes.
  */
 static int i915_probe_agp(struct drm_device *dev, uint32_t *aperture_size,
-                          uint32_t *preallocated_size)
+                          uint32_t *preallocated_size,
+                          uint32_t *start)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u16 tmp = 0;
@@ -1009,10 +1016,159 @@ static int i915_probe_agp(struct drm_device *dev, uint32_t *aperture_size,
                 return -1;
         }
         *preallocated_size = stolen - overhead;
+        *start = overhead;
 
         return 0;
 }
 
+#define PTE_ADDRESS_MASK                0xfffff000
+#define PTE_ADDRESS_MASK_HIGH           0x000000f0 /* i915+ */
+#define PTE_MAPPING_TYPE_UNCACHED       (0 << 1)
+#define PTE_MAPPING_TYPE_DCACHE         (1 << 1) /* i830 only */
+#define PTE_MAPPING_TYPE_CACHED         (3 << 1)
+#define PTE_MAPPING_TYPE_MASK           (3 << 1)
+#define PTE_VALID                       (1 << 0)
+
+/**
+ * i915_gtt_to_phys - take a GTT address and turn it into a physical one
+ * @dev: drm device
+ * @gtt_addr: address to translate
+ *
+ * Some chip functions require allocations from stolen space but need the
+ * physical address of the memory in question.  We use this routine
+ * to get a physical address suitable for register programming from a given
+ * GTT address.
+ */
+static unsigned long i915_gtt_to_phys(struct drm_device *dev,
+                                      unsigned long gtt_addr)
+{
+        unsigned long *gtt;
+        unsigned long entry, phys;
+        int gtt_bar = IS_I9XX(dev) ? 0 : 1;
+        int gtt_offset, gtt_size;
+
+        if (IS_I965G(dev)) {
+                if (IS_G4X(dev) || IS_IGDNG(dev)) {
+                        gtt_offset = 2*1024*1024;
+                        gtt_size = 2*1024*1024;
+                } else {
+                        gtt_offset = 512*1024;
+                        gtt_size = 512*1024;
+                }
+        } else {
+                gtt_bar = 3;
+                gtt_offset = 0;
+                gtt_size = pci_resource_len(dev->pdev, gtt_bar);
+        }
+
+        gtt = ioremap_wc(pci_resource_start(dev->pdev, gtt_bar) + gtt_offset,
+                         gtt_size);
+        if (!gtt) {
+                DRM_ERROR("ioremap of GTT failed\n");
+                return 0;
+        }
+
+        entry = *(volatile u32 *)(gtt + (gtt_addr / 1024));
+
+        DRM_DEBUG("GTT addr: 0x%08lx, PTE: 0x%08lx\n", gtt_addr, entry);
+
+        /* Mask out these reserved bits on this hardware. */
+        if (!IS_I9XX(dev) || IS_I915G(dev) || IS_I915GM(dev) ||
+            IS_I945G(dev) || IS_I945GM(dev)) {
+                entry &= ~PTE_ADDRESS_MASK_HIGH;
+        }
+
+        /* If it's not a mapping type we know, then bail. */
+        if ((entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_UNCACHED &&
+            (entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_CACHED) {
+                iounmap(gtt);
+                return 0;
+        }
+
+        if (!(entry & PTE_VALID)) {
+                DRM_ERROR("bad GTT entry in stolen space\n");
+                iounmap(gtt);
+                return 0;
+        }
+
+        iounmap(gtt);
+
+        phys =(entry & PTE_ADDRESS_MASK) |
+                ((uint64_t)(entry & PTE_ADDRESS_MASK_HIGH) << (32 - 4));
+
+        DRM_DEBUG("GTT addr: 0x%08lx, phys addr: 0x%08lx\n", gtt_addr, phys);
+
+        return phys;
+}
+
+static void i915_warn_stolen(struct drm_device *dev)
+{
+        DRM_ERROR("not enough stolen space for compressed buffer, disabling\n");
+        DRM_ERROR("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
+}
+
+static void i915_setup_compression(struct drm_device *dev, int size)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_mm_node *compressed_fb, *compressed_llb;
+        unsigned long cfb_base, ll_base;
+
+        /* Leave 1M for line length buffer & misc. */
+        compressed_fb = drm_mm_search_free(&dev_priv->vram, size, 4096, 0);
+        if (!compressed_fb) {
+                i915_warn_stolen(dev);
+                return;
+        }
+
+        compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
+        if (!compressed_fb) {
+                i915_warn_stolen(dev);
+                return;
+        }
+
+        cfb_base = i915_gtt_to_phys(dev, compressed_fb->start);
+        if (!cfb_base) {
+                DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
+                drm_mm_put_block(compressed_fb);
+        }
+
+        if (!IS_GM45(dev)) {
+                compressed_llb = drm_mm_search_free(&dev_priv->vram, 4096,
+                                                    4096, 0);
+                if (!compressed_llb) {
+                        i915_warn_stolen(dev);
+                        return;
+                }
+
+                compressed_llb = drm_mm_get_block(compressed_llb, 4096, 4096);
+                if (!compressed_llb) {
+                        i915_warn_stolen(dev);
+                        return;
+                }
+
+                ll_base = i915_gtt_to_phys(dev, compressed_llb->start);
+                if (!ll_base) {
+                        DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
+                        drm_mm_put_block(compressed_fb);
+                        drm_mm_put_block(compressed_llb);
+                }
+        }
+
+        dev_priv->cfb_size = size;
+
+        if (IS_GM45(dev)) {
+                g4x_disable_fbc(dev);
+                I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
+        } else {
+                i8xx_disable_fbc(dev);
+                I915_WRITE(FBC_CFB_BASE, cfb_base);
+                I915_WRITE(FBC_LL_BASE, ll_base);
+        }
+
+        DRM_DEBUG("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", cfb_base,
+                  ll_base, size >> 20);
+}
+
 /* true = enable decode, false = disable decoder */
 static unsigned int i915_vga_set_decode(void *cookie, bool state)
 {
@@ -1027,6 +1183,7 @@ static unsigned int i915_vga_set_decode(void *cookie, bool state)
 }
 
 static int i915_load_modeset_init(struct drm_device *dev,
+                                  unsigned long prealloc_start,
                                   unsigned long prealloc_size,
                                   unsigned long agp_size)
 {
@@ -1047,6 +1204,10 @@ static int i915_load_modeset_init(struct drm_device *dev,
 
         /* Basic memrange allocator for stolen space (aka vram) */
         drm_mm_init(&dev_priv->vram, 0, prealloc_size);
+        DRM_INFO("set up %ldM of stolen space\n", prealloc_size / (1024*1024));
+
+        /* We're off and running w/KMS */
+        dev_priv->mm.suspended = 0;
 
         /* Let GEM Manage from end of prealloc space to end of aperture.
          *
@@ -1059,10 +1220,25 @@ static int i915_load_modeset_init(struct drm_device *dev,
          */
         i915_gem_do_init(dev, prealloc_size, agp_size - 4096);
 
+        mutex_lock(&dev->struct_mutex);
         ret = i915_gem_init_ringbuffer(dev);
+        mutex_unlock(&dev->struct_mutex);
         if (ret)
                 goto out;
 
+        /* Try to set up FBC with a reasonable compressed buffer size */
+        if (IS_MOBILE(dev) && (IS_I9XX(dev) || IS_I965G(dev) || IS_GM45(dev)) &&
+            i915_powersave) {
+                int cfb_size;
+
+                /* Try to get an 8M buffer... */
+                if (prealloc_size > (9*1024*1024))
+                        cfb_size = 8*1024*1024;
+                else /* fall back to 7/8 of the stolen space */
+                        cfb_size = prealloc_size * 7 / 8;
+                i915_setup_compression(dev, cfb_size);
+        }
+
         /* Allow hardware batchbuffers unless told otherwise.
          */
         dev_priv->allow_batchbuffer = 1;
@@ -1180,7 +1356,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         struct drm_i915_private *dev_priv = dev->dev_private;
         resource_size_t base, size;
         int ret = 0, mmio_bar = IS_I9XX(dev) ? 0 : 1;
-        uint32_t agp_size, prealloc_size;
+        uint32_t agp_size, prealloc_size, prealloc_start;
 
         /* i915 has 4 more counters */
         dev->counters += 4;
@@ -1234,7 +1410,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
                          "performance may suffer.\n");
         }
 
-        ret = i915_probe_agp(dev, &agp_size, &prealloc_size);
+        ret = i915_probe_agp(dev, &agp_size, &prealloc_size, &prealloc_start);
         if (ret)
                 goto out_iomapfree;
 
@@ -1300,8 +1476,12 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
                 return ret;
         }
 
+        /* Start out suspended */
+        dev_priv->mm.suspended = 1;
+
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = i915_load_modeset_init(dev, prealloc_size, agp_size);
+                ret = i915_load_modeset_init(dev, prealloc_start,
+                                             prealloc_size, agp_size);
                 if (ret < 0) {
                         DRM_ERROR("failed to init modeset\n");
                         goto out_workqueue_free;
@@ -1313,6 +1493,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         if (!IS_IGDNG(dev))
                 intel_opregion_init(dev, 0);
 
+        setup_timer(&dev_priv->hangcheck_timer, i915_hangcheck_elapsed,
+                    (unsigned long) dev);
         return 0;
 
 out_workqueue_free:
@@ -1333,6 +1515,7 @@ int i915_driver_unload(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         destroy_workqueue(dev_priv->wq);
+        del_timer_sync(&dev_priv->hangcheck_timer);
 
         io_mapping_free(dev_priv->mm.gtt_mapping);
         if (dev_priv->mm.gtt_mtrr >= 0) {
@@ -1472,6 +1655,7 @@ struct drm_ioctl_desc i915_ioctls[] = {
         DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0),
         DRM_IOCTL_DEF(DRM_I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, 0),
         DRM_IOCTL_DEF(DRM_I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
+        DRM_IOCTL_DEF(DRM_I915_GEM_MADVISE, i915_gem_madvise_ioctl, 0),
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index dbe568c9327b..b93814c0d3e2 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -89,6 +89,8 @@ static int i915_suspend(struct drm_device *dev, pm_message_t state)
                 pci_set_power_state(dev->pdev, PCI_D3hot);
         }
 
+        dev_priv->suspended = 1;
+
         return 0;
 }
 
@@ -97,8 +99,6 @@ static int i915_resume(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret = 0;
 
-        pci_set_power_state(dev->pdev, PCI_D0);
-        pci_restore_state(dev->pdev);
         if (pci_enable_device(dev->pdev))
                 return -1;
         pci_set_master(dev->pdev);
@@ -124,9 +124,135 @@ static int i915_resume(struct drm_device *dev)
                 drm_helper_resume_force_mode(dev);
         }
 
+        dev_priv->suspended = 0;
+
         return ret;
 }
 
+/**
+ * i965_reset - reset chip after a hang
+ * @dev: drm device to reset
+ * @flags: reset domains
+ *
+ * Reset the chip.  Useful if a hang is detected. Returns zero on successful
+ * reset or otherwise an error code.
+ *
+ * Procedure is fairly simple:
+ *   - reset the chip using the reset reg
+ *   - re-init context state
+ *   - re-init hardware status page
+ *   - re-init ring buffer
+ *   - re-init interrupt state
+ *   - re-init display
+ */
+int i965_reset(struct drm_device *dev, u8 flags)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        unsigned long timeout;
+        u8 gdrst;
+        /*
+         * We really should only reset the display subsystem if we actually
+         * need to
+         */
+        bool need_display = true;
+
+        mutex_lock(&dev->struct_mutex);
+
+        /*
+         * Clear request list
+         */
+        i915_gem_retire_requests(dev);
+
+        if (need_display)
+                i915_save_display(dev);
+
+        if (IS_I965G(dev) || IS_G4X(dev)) {
+                /*
+                 * Set the domains we want to reset, then the reset bit (bit 0).
+                 * Clear the reset bit after a while and wait for hardware status
+                 * bit (bit 1) to be set
+                 */
+                pci_read_config_byte(dev->pdev, GDRST, &gdrst);
+                pci_write_config_byte(dev->pdev, GDRST, gdrst | flags | ((flags == GDRST_FULL) ? 0x1 : 0x0));
+                udelay(50);
+                pci_write_config_byte(dev->pdev, GDRST, gdrst & 0xfe);
+
+                /* ...we don't want to loop forever though, 500ms should be plenty */
+               timeout = jiffies + msecs_to_jiffies(500);
+                do {
+                        udelay(100);
+                        pci_read_config_byte(dev->pdev, GDRST, &gdrst);
+                } while ((gdrst & 0x1) && time_after(timeout, jiffies));
+
+                if (gdrst & 0x1) {
+                        WARN(true, "i915: Failed to reset chip\n");
+                        mutex_unlock(&dev->struct_mutex);
+                        return -EIO;
+                }
+        } else {
+                DRM_ERROR("Error occurred. Don't know how to reset this chip.\n");
+                return -ENODEV;
+        }
+
+        /* Ok, now get things going again... */
+
+        /*
+         * Everything depends on having the GTT running, so we need to start
+         * there.  Fortunately we don't need to do this unless we reset the
+         * chip at a PCI level.
+         *
+         * Next we need to restore the context, but we don't use those
+         * yet either...
+         *
+         * Ring buffer needs to be re-initialized in the KMS case, or if X
+         * was running at the time of the reset (i.e. we weren't VT
+         * switched away).
+         */
+        if (drm_core_check_feature(dev, DRIVER_MODESET) ||
+            !dev_priv->mm.suspended) {
+                drm_i915_ring_buffer_t *ring = &dev_priv->ring;
+                struct drm_gem_object *obj = ring->ring_obj;
+                struct drm_i915_gem_object *obj_priv = obj->driver_private;
+                dev_priv->mm.suspended = 0;
+
+                /* Stop the ring if it's running. */
+                I915_WRITE(PRB0_CTL, 0);
+                I915_WRITE(PRB0_TAIL, 0);
+                I915_WRITE(PRB0_HEAD, 0);
+
+                /* Initialize the ring. */
+                I915_WRITE(PRB0_START, obj_priv->gtt_offset);
+                I915_WRITE(PRB0_CTL,
+                           ((obj->size - 4096) & RING_NR_PAGES) |
+                           RING_NO_REPORT |
+                           RING_VALID);
+                if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                        i915_kernel_lost_context(dev);
+                else {
+                        ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+                        ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
+                        ring->space = ring->head - (ring->tail + 8);
+                        if (ring->space < 0)
+                                ring->space += ring->Size;
+                }
+
+                mutex_unlock(&dev->struct_mutex);
+                drm_irq_uninstall(dev);
+                drm_irq_install(dev);
+                mutex_lock(&dev->struct_mutex);
+        }
+
+        /*
+         * Display needs restore too...
+         */
+        if (need_display)
+                i915_restore_display(dev);
+
+        mutex_unlock(&dev->struct_mutex);
+        return 0;
+}
+
+
 static int __devinit
 i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
@@ -234,6 +360,8 @@ static int __init i915_init(void)
 {
         driver.num_ioctls = i915_max_ioctl;
 
+        i915_gem_shrinker_init();
+
         /*
          * If CONFIG_DRM_I915_KMS is set, default to KMS unless
          * explicitly disabled with the module pararmeter.
@@ -260,6 +388,7 @@ static int __init i915_init(void)
 
 static void __exit i915_exit(void)
 {
+        i915_gem_shrinker_exit();
         drm_exit(&driver);
 }
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index a0632f8e76ac..b24b2d145b75 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -48,6 +48,11 @@ enum pipe {
         PIPE_B,
 };
 
+enum plane {
+        PLANE_A = 0,
+        PLANE_B,
+};
+
 #define I915_NUM_PIPE   2
 
 /* Interface history:
@@ -148,6 +153,23 @@ struct drm_i915_error_state {
         struct timeval time;
 };
 
+struct drm_i915_display_funcs {
+        void (*dpms)(struct drm_crtc *crtc, int mode);
+        bool (*fbc_enabled)(struct drm_crtc *crtc);
+        void (*enable_fbc)(struct drm_crtc *crtc, unsigned long interval);
+        void (*disable_fbc)(struct drm_device *dev);
+        int (*get_display_clock_speed)(struct drm_device *dev);
+        int (*get_fifo_size)(struct drm_device *dev, int plane);
+        void (*update_wm)(struct drm_device *dev, int planea_clock,
+                          int planeb_clock, int sr_hdisplay, int pixel_size);
+        /* clock updates for mode set */
+        /* cursor updates */
+        /* render clock increase/decrease */
+        /* display clock increase/decrease */
+        /* pll clock increase/decrease */
+        /* clock gating init */
+};
+
 typedef struct drm_i915_private {
         struct drm_device *dev;
 
@@ -198,10 +220,21 @@ typedef struct drm_i915_private {
         unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
         int vblank_pipe;
 
+        /* For hangcheck timer */
+#define DRM_I915_HANGCHECK_PERIOD 75 /* in jiffies */
+        struct timer_list hangcheck_timer;
+        int hangcheck_count;
+        uint32_t last_acthd;
+
         bool cursor_needs_physical;
 
         struct drm_mm vram;
 
+        unsigned long cfb_size;
+        unsigned long cfb_pitch;
+        int cfb_fence;
+        int cfb_plane;
+
         int irq_enabled;
 
         struct intel_opregion opregion;
@@ -222,6 +255,8 @@ typedef struct drm_i915_private {
         unsigned int edp_support:1;
         int lvds_ssc_freq;
 
+        struct notifier_block lid_notifier;
+
         int crt_ddc_bus; /* -1 = unknown, else GPIO to use for CRT DDC */
         struct drm_i915_fence_reg fence_regs[16]; /* assume 965 */
         int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
@@ -234,7 +269,11 @@ typedef struct drm_i915_private {
         struct work_struct error_work;
         struct workqueue_struct *wq;
 
+        /* Display functions */
+        struct drm_i915_display_funcs display;
+
         /* Register state */
+        bool suspended;
         u8 saveLBB;
         u32 saveDSPACNTR;
         u32 saveDSPBCNTR;
@@ -350,6 +389,15 @@ typedef struct drm_i915_private {
                 int gtt_mtrr;
 
                 /**
+                 * Membership on list of all loaded devices, used to evict
+                 * inactive buffers under memory pressure.
+                 *
+                 * Modifications should only be done whilst holding the
+                 * shrink_list_lock spinlock.
+                 */
+                struct list_head shrink_list;
+
+                /**
                  * List of objects currently involved in rendering from the
                  * ringbuffer.
                  *
@@ -432,7 +480,7 @@ typedef struct drm_i915_private {
                  * It prevents command submission from occuring and makes
                  * every pending request fail
                  */
-                int wedged;
+                atomic_t wedged;
 
                 /** Bit 6 swizzling required for X tiling */
                 uint32_t bit_6_swizzle_x;
@@ -491,10 +539,7 @@ struct drm_i915_gem_object {
          * This is the same as gtt_space->start
          */
         uint32_t gtt_offset;
-        /**
-         * Required alignment for the object
-         */
-        uint32_t gtt_alignment;
+
         /**
          * Fake offset for use by mmap(2)
          */
@@ -541,6 +586,11 @@ struct drm_i915_gem_object {
          * in an execbuffer object list.
          */
         int in_execbuffer;
+
+        /**
+         * Advice: are the backing pages purgeable?
+         */
+        int madv;
 };
 
 /**
@@ -585,6 +635,8 @@ extern int i915_max_ioctl;
 extern unsigned int i915_fbpercrtc;
 extern unsigned int i915_powersave;
 
+extern void i915_save_display(struct drm_device *dev);
+extern void i915_restore_display(struct drm_device *dev);
 extern int i915_master_create(struct drm_device *dev, struct drm_master *master);
 extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master);
 
@@ -604,8 +656,10 @@ extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
 extern int i915_emit_box(struct drm_device *dev,
                          struct drm_clip_rect *boxes,
                          int i, int DR1, int DR4);
+extern int i965_reset(struct drm_device *dev, u8 flags);
 
 /* i915_irq.c */
+void i915_hangcheck_elapsed(unsigned long data);
 extern int i915_irq_emit(struct drm_device *dev, void *data,
                          struct drm_file *file_priv);
 extern int i915_irq_wait(struct drm_device *dev, void *data,
@@ -676,6 +730,8 @@ int i915_gem_busy_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *file_priv);
 int i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
                             struct drm_file *file_priv);
+int i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
+                           struct drm_file *file_priv);
 int i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
                            struct drm_file *file_priv);
 int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
@@ -695,6 +751,7 @@ int i915_gem_object_unbind(struct drm_gem_object *obj);
 void i915_gem_release_mmap(struct drm_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 uint32_t i915_get_gem_seqno(struct drm_device *dev);
+bool i915_seqno_passed(uint32_t seq1, uint32_t seq2);
 int i915_gem_object_get_fence_reg(struct drm_gem_object *obj);
 int i915_gem_object_put_fence_reg(struct drm_gem_object *obj);
 void i915_gem_retire_requests(struct drm_device *dev);
@@ -720,6 +777,9 @@ int i915_gem_object_get_pages(struct drm_gem_object *obj);
 void i915_gem_object_put_pages(struct drm_gem_object *obj);
 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv);
 
+void i915_gem_shrinker_init(void);
+void i915_gem_shrinker_exit(void);
+
 /* i915_gem_tiling.c */
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
 void i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj);
@@ -767,6 +827,8 @@ static inline void opregion_enable_asle(struct drm_device *dev) { return; }
 extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
+extern void i8xx_disable_fbc(struct drm_device *dev);
+extern void g4x_disable_fbc(struct drm_device *dev);
 
 /**
  * Lock test for when it's just for synchronization of ring access.
@@ -864,6 +926,7 @@ extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
                        (dev)->pci_device == 0x2E12 || \
                        (dev)->pci_device == 0x2E22 || \
                        (dev)->pci_device == 0x2E32 || \
+                       (dev)->pci_device == 0x2E42 || \
                        (dev)->pci_device == 0x0042 || \
                        (dev)->pci_device == 0x0046)
 
@@ -876,6 +939,7 @@ extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
                      (dev)->pci_device == 0x2E12 || \
                      (dev)->pci_device == 0x2E22 || \
                      (dev)->pci_device == 0x2E32 || \
+                     (dev)->pci_device == 0x2E42 || \
                      IS_GM45(dev))
 
 #define IS_IGDG(dev) ((dev)->pci_device == 0xa001)
@@ -909,12 +973,13 @@ extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
 #define SUPPORTS_INTEGRATED_HDMI(dev)   (IS_G4X(dev) || IS_IGDNG(dev))
 #define SUPPORTS_INTEGRATED_DP(dev)     (IS_G4X(dev) || IS_IGDNG(dev))
 #define SUPPORTS_EDP(dev)               (IS_IGDNG_M(dev))
-#define I915_HAS_HOTPLUG(dev) (IS_I945G(dev) || IS_I945GM(dev) || IS_I965G(dev))
+#define I915_HAS_HOTPLUG(dev) (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev) || IS_I965G(dev))
 /* dsparb controlled by hw only */
 #define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IGDNG(dev))
 
 #define HAS_FW_BLC(dev) (IS_I9XX(dev) || IS_G4X(dev) || IS_IGDNG(dev))
 #define HAS_PIPE_CXSR(dev) (IS_G4X(dev) || IS_IGDNG(dev))
+#define I915_HAS_FBC(dev) (IS_MOBILE(dev) && (IS_I9XX(dev) || IS_I965G(dev)))
 
 #define PRIMARY_RINGBUFFER_SIZE         (128*1024)
 
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index c67317112f4a..40727d4c2919 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -29,6 +29,7 @@
 #include "drm.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
+#include "i915_trace.h"
 #include "intel_drv.h"
 #include <linux/swap.h>
 #include <linux/pci.h>
@@ -48,11 +49,15 @@ static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
                                            unsigned alignment);
 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
-static int i915_gem_evict_something(struct drm_device *dev);
+static int i915_gem_evict_something(struct drm_device *dev, int min_size);
+static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
                                 struct drm_i915_gem_pwrite *args,
                                 struct drm_file *file_priv);
 
+static LIST_HEAD(shrink_list);
+static DEFINE_SPINLOCK(shrink_list_lock);
+
 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
                      unsigned long end)
 {
@@ -316,6 +321,45 @@ fail_unlock:
         return ret;
 }
 
+static inline gfp_t
+i915_gem_object_get_page_gfp_mask (struct drm_gem_object *obj)
+{
+        return mapping_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping);
+}
+
+static inline void
+i915_gem_object_set_page_gfp_mask (struct drm_gem_object *obj, gfp_t gfp)
+{
+        mapping_set_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping, gfp);
+}
+
+static int
+i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
+{
+        int ret;
+
+        ret = i915_gem_object_get_pages(obj);
+
+        /* If we've insufficient memory to map in the pages, attempt
+         * to make some space by throwing out some old buffers.
+         */
+        if (ret == -ENOMEM) {
+                struct drm_device *dev = obj->dev;
+                gfp_t gfp;
+
+                ret = i915_gem_evict_something(dev, obj->size);
+                if (ret)
+                        return ret;
+
+                gfp = i915_gem_object_get_page_gfp_mask(obj);
+                i915_gem_object_set_page_gfp_mask(obj, gfp & ~__GFP_NORETRY);
+                ret = i915_gem_object_get_pages(obj);
+                i915_gem_object_set_page_gfp_mask (obj, gfp);
+        }
+
+        return ret;
+}
+
 /**
  * This is the fallback shmem pread path, which allocates temporary storage
  * in kernel space to copy_to_user into outside of the struct_mutex, so we
@@ -367,8 +411,8 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 
         mutex_lock(&dev->struct_mutex);
 
-        ret = i915_gem_object_get_pages(obj);
-        if (ret != 0)
+        ret = i915_gem_object_get_pages_or_evict(obj);
+        if (ret)
                 goto fail_unlock;
 
         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
@@ -842,8 +886,8 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 
         mutex_lock(&dev->struct_mutex);
 
-        ret = i915_gem_object_get_pages(obj);
-        if (ret != 0)
+        ret = i915_gem_object_get_pages_or_evict(obj);
+        if (ret)
                 goto fail_unlock;
 
         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
@@ -1155,28 +1199,22 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         /* Now bind it into the GTT if needed */
         mutex_lock(&dev->struct_mutex);
         if (!obj_priv->gtt_space) {
-                ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
-                if (ret) {
-                        mutex_unlock(&dev->struct_mutex);
-                        return VM_FAULT_SIGBUS;
-                }
-
-                ret = i915_gem_object_set_to_gtt_domain(obj, write);
-                if (ret) {
-                        mutex_unlock(&dev->struct_mutex);
-                        return VM_FAULT_SIGBUS;
-                }
+                ret = i915_gem_object_bind_to_gtt(obj, 0);
+                if (ret)
+                        goto unlock;
 
                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
+                ret = i915_gem_object_set_to_gtt_domain(obj, write);
+                if (ret)
+                        goto unlock;
         }
 
         /* Need a new fence register? */
         if (obj_priv->tiling_mode != I915_TILING_NONE) {
                 ret = i915_gem_object_get_fence_reg(obj);
-                if (ret) {
-                        mutex_unlock(&dev->struct_mutex);
-                        return VM_FAULT_SIGBUS;
-                }
+                if (ret)
+                        goto unlock;
         }
 
         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
@@ -1184,18 +1222,18 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
         /* Finally, remap it using the new GTT offset */
         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
-
+unlock:
         mutex_unlock(&dev->struct_mutex);
 
         switch (ret) {
+        case 0:
+        case -ERESTARTSYS:
+                return VM_FAULT_NOPAGE;
         case -ENOMEM:
         case -EAGAIN:
                 return VM_FAULT_OOM;
-        case -EFAULT:
-        case -EINVAL:
-                return VM_FAULT_SIGBUS;
         default:
-                return VM_FAULT_NOPAGE;
+                return VM_FAULT_SIGBUS;
         }
 }
 
@@ -1388,6 +1426,14 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 
         obj_priv = obj->driver_private;
 
+        if (obj_priv->madv != I915_MADV_WILLNEED) {
+                DRM_ERROR("Attempting to mmap a purgeable buffer\n");
+                drm_gem_object_unreference(obj);
+                mutex_unlock(&dev->struct_mutex);
+                return -EINVAL;
+        }
+
+
         if (!obj_priv->mmap_offset) {
                 ret = i915_gem_create_mmap_offset(obj);
                 if (ret) {
@@ -1399,22 +1445,12 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 
         args->offset = obj_priv->mmap_offset;
 
-        obj_priv->gtt_alignment = i915_gem_get_gtt_alignment(obj);
-
-        /* Make sure the alignment is correct for fence regs etc */
-        if (obj_priv->agp_mem &&
-            (obj_priv->gtt_offset & (obj_priv->gtt_alignment - 1))) {
-                drm_gem_object_unreference(obj);
-                mutex_unlock(&dev->struct_mutex);
-                return -EINVAL;
-        }
-
         /*
          * Pull it into the GTT so that we have a page list (makes the
          * initial fault faster and any subsequent flushing possible).
          */
         if (!obj_priv->agp_mem) {
-                ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
+                ret = i915_gem_object_bind_to_gtt(obj, 0);
                 if (ret) {
                         drm_gem_object_unreference(obj);
                         mutex_unlock(&dev->struct_mutex);
@@ -1437,6 +1473,7 @@ i915_gem_object_put_pages(struct drm_gem_object *obj)
         int i;
 
         BUG_ON(obj_priv->pages_refcount == 0);
+        BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
 
         if (--obj_priv->pages_refcount != 0)
                 return;
@@ -1444,13 +1481,21 @@ i915_gem_object_put_pages(struct drm_gem_object *obj)
         if (obj_priv->tiling_mode != I915_TILING_NONE)
                 i915_gem_object_save_bit_17_swizzle(obj);
 
-        for (i = 0; i < page_count; i++)
-                if (obj_priv->pages[i] != NULL) {
-                        if (obj_priv->dirty)
-                                set_page_dirty(obj_priv->pages[i]);
+        if (obj_priv->madv == I915_MADV_DONTNEED)
+                obj_priv->dirty = 0;
+
+        for (i = 0; i < page_count; i++) {
+                if (obj_priv->pages[i] == NULL)
+                        break;
+
+                if (obj_priv->dirty)
+                        set_page_dirty(obj_priv->pages[i]);
+
+                if (obj_priv->madv == I915_MADV_WILLNEED)
                         mark_page_accessed(obj_priv->pages[i]);
-                        page_cache_release(obj_priv->pages[i]);
-                }
+
+                page_cache_release(obj_priv->pages[i]);
+        }
         obj_priv->dirty = 0;
 
         drm_free_large(obj_priv->pages);
@@ -1489,6 +1534,26 @@ i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
         obj_priv->last_rendering_seqno = 0;
 }
 
+/* Immediately discard the backing storage */
+static void
+i915_gem_object_truncate(struct drm_gem_object *obj)
+{
+        struct drm_i915_gem_object *obj_priv = obj->driver_private;
+        struct inode *inode;
+
+        inode = obj->filp->f_path.dentry->d_inode;
+        if (inode->i_op->truncate)
+                inode->i_op->truncate (inode);
+
+        obj_priv->madv = __I915_MADV_PURGED;
+}
+
+static inline int
+i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
+{
+        return obj_priv->madv == I915_MADV_DONTNEED;
+}
+
 static void
 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
 {
@@ -1577,15 +1642,24 @@ i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
 
                         if ((obj->write_domain & flush_domains) ==
                             obj->write_domain) {
+                                uint32_t old_write_domain = obj->write_domain;
+
                                 obj->write_domain = 0;
                                 i915_gem_object_move_to_active(obj, seqno);
+
+                                trace_i915_gem_object_change_domain(obj,
+                                                                    obj->read_domains,
+                                                                    old_write_domain);
                         }
                 }
 
         }
 
-        if (was_empty && !dev_priv->mm.suspended)
-                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+        if (!dev_priv->mm.suspended) {
+                mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
+                if (was_empty)
+                        queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+        }
         return seqno;
 }
 
@@ -1623,6 +1697,8 @@ i915_gem_retire_request(struct drm_device *dev,
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
 
+        trace_i915_gem_request_retire(dev, request->seqno);
+
         /* Move any buffers on the active list that are no longer referenced
          * by the ringbuffer to the flushing/inactive lists as appropriate.
          */
@@ -1671,7 +1747,7 @@ out:
 /**
  * Returns true if seq1 is later than seq2.
  */
-static int
+bool
 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
 {
         return (int32_t)(seq1 - seq2) >= 0;
@@ -1709,7 +1785,7 @@ i915_gem_retire_requests(struct drm_device *dev)
                 retiring_seqno = request->seqno;
 
                 if (i915_seqno_passed(seqno, retiring_seqno) ||
-                    dev_priv->mm.wedged) {
+                    atomic_read(&dev_priv->mm.wedged)) {
                         i915_gem_retire_request(dev, request);
 
                         list_del(&request->list);
@@ -1751,6 +1827,9 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno)
 
         BUG_ON(seqno == 0);
 
+        if (atomic_read(&dev_priv->mm.wedged))
+                return -EIO;
+
         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
                 if (IS_IGDNG(dev))
                         ier = I915_READ(DEIER) | I915_READ(GTIER);
@@ -1763,16 +1842,20 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno)
                         i915_driver_irq_postinstall(dev);
                 }
 
+                trace_i915_gem_request_wait_begin(dev, seqno);
+
                 dev_priv->mm.waiting_gem_seqno = seqno;
                 i915_user_irq_get(dev);
                 ret = wait_event_interruptible(dev_priv->irq_queue,
                                                i915_seqno_passed(i915_get_gem_seqno(dev),
                                                                  seqno) ||
-                                               dev_priv->mm.wedged);
+                                               atomic_read(&dev_priv->mm.wedged));
                 i915_user_irq_put(dev);
                 dev_priv->mm.waiting_gem_seqno = 0;
+
+                trace_i915_gem_request_wait_end(dev, seqno);
         }
-        if (dev_priv->mm.wedged)
+        if (atomic_read(&dev_priv->mm.wedged))
                 ret = -EIO;
 
         if (ret && ret != -ERESTARTSYS)
@@ -1803,6 +1886,8 @@ i915_gem_flush(struct drm_device *dev,
         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
                   invalidate_domains, flush_domains);
 #endif
+        trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
+                                     invalidate_domains, flush_domains);
 
         if (flush_domains & I915_GEM_DOMAIN_CPU)
                 drm_agp_chipset_flush(dev);
@@ -1915,6 +2000,12 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
                 return -EINVAL;
         }
 
+        /* blow away mappings if mapped through GTT */
+        i915_gem_release_mmap(obj);
+
+        if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
+                i915_gem_clear_fence_reg(obj);
+
         /* Move the object to the CPU domain to ensure that
          * any possible CPU writes while it's not in the GTT
          * are flushed when we go to remap it. This will
@@ -1928,21 +2019,16 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
                 return ret;
         }
 
+        BUG_ON(obj_priv->active);
+
         if (obj_priv->agp_mem != NULL) {
                 drm_unbind_agp(obj_priv->agp_mem);
                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
                 obj_priv->agp_mem = NULL;
         }
 
-        BUG_ON(obj_priv->active);
-
-        /* blow away mappings if mapped through GTT */
-        i915_gem_release_mmap(obj);
-
-        if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-                i915_gem_clear_fence_reg(obj);
-
         i915_gem_object_put_pages(obj);
+        BUG_ON(obj_priv->pages_refcount);
 
         if (obj_priv->gtt_space) {
                 atomic_dec(&dev->gtt_count);
@@ -1956,40 +2042,113 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
         if (!list_empty(&obj_priv->list))
                 list_del_init(&obj_priv->list);
 
+        if (i915_gem_object_is_purgeable(obj_priv))
+                i915_gem_object_truncate(obj);
+
+        trace_i915_gem_object_unbind(obj);
+
         return 0;
 }
 
+static struct drm_gem_object *
+i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj_priv;
+        struct drm_gem_object *best = NULL;
+        struct drm_gem_object *first = NULL;
+
+        /* Try to find the smallest clean object */
+        list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
+                struct drm_gem_object *obj = obj_priv->obj;
+                if (obj->size >= min_size) {
+                        if ((!obj_priv->dirty ||
+                             i915_gem_object_is_purgeable(obj_priv)) &&
+                            (!best || obj->size < best->size)) {
+                                best = obj;
+                                if (best->size == min_size)
+                                        return best;
+                        }
+                        if (!first)
+                            first = obj;
+                }
+        }
+
+        return best ? best : first;
+}
+
 static int
-i915_gem_evict_something(struct drm_device *dev)
+i915_gem_evict_everything(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        uint32_t seqno;
+        int ret;
+        bool lists_empty;
+
+        spin_lock(&dev_priv->mm.active_list_lock);
+        lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
+                       list_empty(&dev_priv->mm.flushing_list) &&
+                       list_empty(&dev_priv->mm.active_list));
+        spin_unlock(&dev_priv->mm.active_list_lock);
+
+        if (lists_empty)
+                return -ENOSPC;
+
+        /* Flush everything (on to the inactive lists) and evict */
+        i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+        seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
+        if (seqno == 0)
+                return -ENOMEM;
+
+        ret = i915_wait_request(dev, seqno);
+        if (ret)
+                return ret;
+
+        ret = i915_gem_evict_from_inactive_list(dev);
+        if (ret)
+                return ret;
+
+        spin_lock(&dev_priv->mm.active_list_lock);
+        lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
+                       list_empty(&dev_priv->mm.flushing_list) &&
+                       list_empty(&dev_priv->mm.active_list));
+        spin_unlock(&dev_priv->mm.active_list_lock);
+        BUG_ON(!lists_empty);
+
+        return 0;
+}
+
+static int
+i915_gem_evict_something(struct drm_device *dev, int min_size)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret = 0;
+        int ret;
 
         for (;;) {
+                i915_gem_retire_requests(dev);
+
                 /* If there's an inactive buffer available now, grab it
                  * and be done.
                  */
-                if (!list_empty(&dev_priv->mm.inactive_list)) {
-                        obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
-                                                    struct drm_i915_gem_object,
-                                                    list);
-                        obj = obj_priv->obj;
-                        BUG_ON(obj_priv->pin_count != 0);
+                obj = i915_gem_find_inactive_object(dev, min_size);
+                if (obj) {
+                        struct drm_i915_gem_object *obj_priv;
+
 #if WATCH_LRU
                         DRM_INFO("%s: evicting %p\n", __func__, obj);
 #endif
+                        obj_priv = obj->driver_private;
+                        BUG_ON(obj_priv->pin_count != 0);
                         BUG_ON(obj_priv->active);
 
                         /* Wait on the rendering and unbind the buffer. */
-                        ret = i915_gem_object_unbind(obj);
-                        break;
+                        return i915_gem_object_unbind(obj);
                 }
 
                 /* If we didn't get anything, but the ring is still processing
-                 * things, wait for one of those things to finish and hopefully
-                 * leave us a buffer to evict.
+                 * things, wait for the next to finish and hopefully leave us
+                 * a buffer to evict.
                  */
                 if (!list_empty(&dev_priv->mm.request_list)) {
                         struct drm_i915_gem_request *request;
@@ -2000,16 +2159,9 @@ i915_gem_evict_something(struct drm_device *dev)
 
                         ret = i915_wait_request(dev, request->seqno);
                         if (ret)
-                                break;
+                                return ret;
 
-                        /* if waiting caused an object to become inactive,
-                         * then loop around and wait for it. Otherwise, we
-                         * assume that waiting freed and unbound something,
-                         * so there should now be some space in the GTT
-                         */
-                        if (!list_empty(&dev_priv->mm.inactive_list))
-                                continue;
-                        break;
+                        continue;
                 }
 
                 /* If we didn't have anything on the request list but there
@@ -2018,46 +2170,44 @@ i915_gem_evict_something(struct drm_device *dev)
                  * will get moved to inactive.
                  */
                 if (!list_empty(&dev_priv->mm.flushing_list)) {
-                        obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
-                                                    struct drm_i915_gem_object,
-                                                    list);
-                        obj = obj_priv->obj;
+                        struct drm_i915_gem_object *obj_priv;
 
-                        i915_gem_flush(dev,
-                                       obj->write_domain,
-                                       obj->write_domain);
-                        i915_add_request(dev, NULL, obj->write_domain);
+                        /* Find an object that we can immediately reuse */
+                        list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
+                                obj = obj_priv->obj;
+                                if (obj->size >= min_size)
+                                        break;
 
-                        obj = NULL;
-                        continue;
-                }
+                                obj = NULL;
+                        }
 
-                DRM_ERROR("inactive empty %d request empty %d "
-                          "flushing empty %d\n",
-                          list_empty(&dev_priv->mm.inactive_list),
-                          list_empty(&dev_priv->mm.request_list),
-                          list_empty(&dev_priv->mm.flushing_list));
-                /* If we didn't do any of the above, there's nothing to be done
-                 * and we just can't fit it in.
-                 */
-                return -ENOSPC;
-        }
-        return ret;
-}
+                        if (obj != NULL) {
+                                uint32_t seqno;
 
-static int
-i915_gem_evict_everything(struct drm_device *dev)
-{
-        int ret;
+                                i915_gem_flush(dev,
+                                               obj->write_domain,
+                                               obj->write_domain);
+                                seqno = i915_add_request(dev, NULL, obj->write_domain);
+                                if (seqno == 0)
+                                        return -ENOMEM;
 
-        for (;;) {
-                ret = i915_gem_evict_something(dev);
-                if (ret != 0)
-                        break;
+                                ret = i915_wait_request(dev, seqno);
+                                if (ret)
+                                        return ret;
+
+                                continue;
+                        }
+                }
+
+                /* If we didn't do any of the above, there's no single buffer
+                 * large enough to swap out for the new one, so just evict
+                 * everything and start again. (This should be rare.)
+                 */
+                if (!list_empty (&dev_priv->mm.inactive_list))
+                        return i915_gem_evict_from_inactive_list(dev);
+                else
+                        return i915_gem_evict_everything(dev);
         }
-        if (ret == -ENOSPC)
-                return 0;
-        return ret;
 }
 
 int
@@ -2080,7 +2230,6 @@ i915_gem_object_get_pages(struct drm_gem_object *obj)
         BUG_ON(obj_priv->pages != NULL);
         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
         if (obj_priv->pages == NULL) {
-                DRM_ERROR("Faled to allocate page list\n");
                 obj_priv->pages_refcount--;
                 return -ENOMEM;
         }
@@ -2091,7 +2240,6 @@ i915_gem_object_get_pages(struct drm_gem_object *obj)
                 page = read_mapping_page(mapping, i, NULL);
                 if (IS_ERR(page)) {
                         ret = PTR_ERR(page);
-                        DRM_ERROR("read_mapping_page failed: %d\n", ret);
                         i915_gem_object_put_pages(obj);
                         return ret;
                 }
@@ -2328,6 +2476,8 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
         else
                 i830_write_fence_reg(reg);
 
+        trace_i915_gem_object_get_fence(obj, i, obj_priv->tiling_mode);
+
         return 0;
 }
 
@@ -2410,10 +2560,17 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj_priv = obj->driver_private;
         struct drm_mm_node *free_space;
-        int page_count, ret;
+        bool retry_alloc = false;
+        int ret;
 
         if (dev_priv->mm.suspended)
                 return -EBUSY;
+
+        if (obj_priv->madv != I915_MADV_WILLNEED) {
+                DRM_ERROR("Attempting to bind a purgeable object\n");
+                return -EINVAL;
+        }
+
         if (alignment == 0)
                 alignment = i915_gem_get_gtt_alignment(obj);
         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
@@ -2433,30 +2590,16 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
                 }
         }
         if (obj_priv->gtt_space == NULL) {
-                bool lists_empty;
-
                 /* If the gtt is empty and we're still having trouble
                  * fitting our object in, we're out of memory.
                  */
 #if WATCH_LRU
                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
 #endif
-                spin_lock(&dev_priv->mm.active_list_lock);
-                lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
-                               list_empty(&dev_priv->mm.flushing_list) &&
-                               list_empty(&dev_priv->mm.active_list));
-                spin_unlock(&dev_priv->mm.active_list_lock);
-                if (lists_empty) {
-                        DRM_ERROR("GTT full, but LRU list empty\n");
-                        return -ENOSPC;
-                }
-
-                ret = i915_gem_evict_something(dev);
-                if (ret != 0) {
-                        if (ret != -ERESTARTSYS)
-                                DRM_ERROR("Failed to evict a buffer %d\n", ret);
+                ret = i915_gem_evict_something(dev, obj->size);
+                if (ret)
                         return ret;
-                }
+
                 goto search_free;
         }
 
@@ -2464,27 +2607,56 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
         DRM_INFO("Binding object of size %zd at 0x%08x\n",
                  obj->size, obj_priv->gtt_offset);
 #endif
+        if (retry_alloc) {
+                i915_gem_object_set_page_gfp_mask (obj,
+                                                   i915_gem_object_get_page_gfp_mask (obj) & ~__GFP_NORETRY);
+        }
         ret = i915_gem_object_get_pages(obj);
+        if (retry_alloc) {
+                i915_gem_object_set_page_gfp_mask (obj,
+                                                   i915_gem_object_get_page_gfp_mask (obj) | __GFP_NORETRY);
+        }
         if (ret) {
                 drm_mm_put_block(obj_priv->gtt_space);
                 obj_priv->gtt_space = NULL;
+
+                if (ret == -ENOMEM) {
+                        /* first try to clear up some space from the GTT */
+                        ret = i915_gem_evict_something(dev, obj->size);
+                        if (ret) {
+                                /* now try to shrink everyone else */
+                                if (! retry_alloc) {
+                                    retry_alloc = true;
+                                    goto search_free;
+                                }
+
+                                return ret;
+                        }
+
+                        goto search_free;
+                }
+
                 return ret;
         }
 
-        page_count = obj->size / PAGE_SIZE;
         /* Create an AGP memory structure pointing at our pages, and bind it
          * into the GTT.
          */
         obj_priv->agp_mem = drm_agp_bind_pages(dev,
                                                obj_priv->pages,
-                                               page_count,
+                                               obj->size >> PAGE_SHIFT,
                                                obj_priv->gtt_offset,
                                                obj_priv->agp_type);
         if (obj_priv->agp_mem == NULL) {
                 i915_gem_object_put_pages(obj);
                 drm_mm_put_block(obj_priv->gtt_space);
                 obj_priv->gtt_space = NULL;
-                return -ENOMEM;
+
+                ret = i915_gem_evict_something(dev, obj->size);
+                if (ret)
+                        return ret;
+
+                goto search_free;
         }
         atomic_inc(&dev->gtt_count);
         atomic_add(obj->size, &dev->gtt_memory);
@@ -2496,6 +2668,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
 
+        trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
+
         return 0;
 }
 
@@ -2511,15 +2685,7 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
         if (obj_priv->pages == NULL)
                 return;
 
-        /* XXX: The 865 in particular appears to be weird in how it handles
-         * cache flushing.  We haven't figured it out, but the
-         * clflush+agp_chipset_flush doesn't appear to successfully get the
-         * data visible to the PGU, while wbinvd + agp_chipset_flush does.
-         */
-        if (IS_I865G(obj->dev)) {
-                wbinvd();
-                return;
-        }
+        trace_i915_gem_object_clflush(obj);
 
         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
 }
@@ -2530,21 +2696,29 @@ i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
 {
         struct drm_device *dev = obj->dev;
         uint32_t seqno;
+        uint32_t old_write_domain;
 
         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
                 return;
 
         /* Queue the GPU write cache flushing we need. */
+        old_write_domain = obj->write_domain;
         i915_gem_flush(dev, 0, obj->write_domain);
         seqno = i915_add_request(dev, NULL, obj->write_domain);
         obj->write_domain = 0;
         i915_gem_object_move_to_active(obj, seqno);
+
+        trace_i915_gem_object_change_domain(obj,
+                                            obj->read_domains,
+                                            old_write_domain);
 }
 
 /** Flushes the GTT write domain for the object if it's dirty. */
 static void
 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
 {
+        uint32_t old_write_domain;
+
         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
                 return;
 
@@ -2552,7 +2726,12 @@ i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
          * to it immediately go to main memory as far as we know, so there's
          * no chipset flush.  It also doesn't land in render cache.
          */
+        old_write_domain = obj->write_domain;
         obj->write_domain = 0;
+
+        trace_i915_gem_object_change_domain(obj,
+                                            obj->read_domains,
+                                            old_write_domain);
 }
 
 /** Flushes the CPU write domain for the object if it's dirty. */
@@ -2560,13 +2739,19 @@ static void
 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
 {
         struct drm_device *dev = obj->dev;
+        uint32_t old_write_domain;
 
         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
                 return;
 
         i915_gem_clflush_object(obj);
         drm_agp_chipset_flush(dev);
+        old_write_domain = obj->write_domain;
         obj->write_domain = 0;
+
+        trace_i915_gem_object_change_domain(obj,
+                                            obj->read_domains,
+                                            old_write_domain);
 }
 
 /**
@@ -2579,6 +2764,7 @@ int
 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
 {
         struct drm_i915_gem_object *obj_priv = obj->driver_private;
+        uint32_t old_write_domain, old_read_domains;
         int ret;
 
         /* Not valid to be called on unbound objects. */
@@ -2591,6 +2777,9 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
         if (ret != 0)
                 return ret;
 
+        old_write_domain = obj->write_domain;
+        old_read_domains = obj->read_domains;
+
         /* If we're writing through the GTT domain, then CPU and GPU caches
          * will need to be invalidated at next use.
          */
@@ -2609,6 +2798,10 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
                 obj_priv->dirty = 1;
         }
 
+        trace_i915_gem_object_change_domain(obj,
+                                            old_read_domains,
+                                            old_write_domain);
+
         return 0;
 }
 
@@ -2621,6 +2814,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
 static int
 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
 {
+        uint32_t old_write_domain, old_read_domains;
         int ret;
 
         i915_gem_object_flush_gpu_write_domain(obj);
@@ -2636,6 +2830,9 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
          */
         i915_gem_object_set_to_full_cpu_read_domain(obj);
 
+        old_write_domain = obj->write_domain;
+        old_read_domains = obj->read_domains;
+
         /* Flush the CPU cache if it's still invalid. */
         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
                 i915_gem_clflush_object(obj);
@@ -2656,6 +2853,10 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
                 obj->write_domain = I915_GEM_DOMAIN_CPU;
         }
 
+        trace_i915_gem_object_change_domain(obj,
+                                            old_read_domains,
+                                            old_write_domain);
+
         return 0;
 }
 
@@ -2777,6 +2978,7 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
         struct drm_i915_gem_object      *obj_priv = obj->driver_private;
         uint32_t                        invalidate_domains = 0;
         uint32_t                        flush_domains = 0;
+        uint32_t                        old_read_domains;
 
         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
@@ -2823,6 +3025,8 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
                 i915_gem_clflush_object(obj);
         }
 
+        old_read_domains = obj->read_domains;
+
         /* The actual obj->write_domain will be updated with
          * pending_write_domain after we emit the accumulated flush for all
          * of our domain changes in execbuffers (which clears objects'
@@ -2841,6 +3045,10 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
                  obj->read_domains, obj->write_domain,
                  dev->invalidate_domains, dev->flush_domains);
 #endif
+
+        trace_i915_gem_object_change_domain(obj,
+                                            old_read_domains,
+                                            obj->write_domain);
 }
 
 /**
@@ -2893,6 +3101,7 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
                                           uint64_t offset, uint64_t size)
 {
         struct drm_i915_gem_object *obj_priv = obj->driver_private;
+        uint32_t old_read_domains;
         int i, ret;
 
         if (offset == 0 && size == obj->size)
@@ -2939,8 +3148,13 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
          */
         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
+        old_read_domains = obj->read_domains;
         obj->read_domains |= I915_GEM_DOMAIN_CPU;
 
+        trace_i915_gem_object_change_domain(obj,
+                                            old_read_domains,
+                                            obj->write_domain);
+
         return 0;
 }
 
@@ -2984,6 +3198,21 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
                 }
                 target_obj_priv = target_obj->driver_private;
 
+#if WATCH_RELOC
+                DRM_INFO("%s: obj %p offset %08x target %d "
+                         "read %08x write %08x gtt %08x "
+                         "presumed %08x delta %08x\n",
+                         __func__,
+                         obj,
+                         (int) reloc->offset,
+                         (int) reloc->target_handle,
+                         (int) reloc->read_domains,
+                         (int) reloc->write_domain,
+                         (int) target_obj_priv->gtt_offset,
+                         (int) reloc->presumed_offset,
+                         reloc->delta);
+#endif
+
                 /* The target buffer should have appeared before us in the
                  * exec_object list, so it should have a GTT space bound by now.
                  */
@@ -2995,25 +3224,7 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
                         return -EINVAL;
                 }
 
-                if (reloc->offset > obj->size - 4) {
-                        DRM_ERROR("Relocation beyond object bounds: "
-                                  "obj %p target %d offset %d size %d.\n",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset, (int) obj->size);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-                if (reloc->offset & 3) {
-                        DRM_ERROR("Relocation not 4-byte aligned: "
-                                  "obj %p target %d offset %d.\n",
-                                  obj, reloc->target_handle,
-                                  (int) reloc->offset);
-                        drm_gem_object_unreference(target_obj);
-                        i915_gem_object_unpin(obj);
-                        return -EINVAL;
-                }
-
+                /* Validate that the target is in a valid r/w GPU domain */
                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
                         DRM_ERROR("reloc with read/write CPU domains: "
@@ -3027,7 +3238,6 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
                         i915_gem_object_unpin(obj);
                         return -EINVAL;
                 }
-
                 if (reloc->write_domain && target_obj->pending_write_domain &&
                     reloc->write_domain != target_obj->pending_write_domain) {
                         DRM_ERROR("Write domain conflict: "
@@ -3042,21 +3252,6 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
                         return -EINVAL;
                 }
 
-#if WATCH_RELOC
-                DRM_INFO("%s: obj %p offset %08x target %d "
-                         "read %08x write %08x gtt %08x "
-                         "presumed %08x delta %08x\n",
-                         __func__,
-                         obj,
-                         (int) reloc->offset,
-                         (int) reloc->target_handle,
-                         (int) reloc->read_domains,
-                         (int) reloc->write_domain,
-                         (int) target_obj_priv->gtt_offset,
-                         (int) reloc->presumed_offset,
-                         reloc->delta);
-#endif
-
                 target_obj->pending_read_domains |= reloc->read_domains;
                 target_obj->pending_write_domain |= reloc->write_domain;
 
@@ -3068,6 +3263,37 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
                         continue;
                 }
 
+                /* Check that the relocation address is valid... */
+                if (reloc->offset > obj->size - 4) {
+                        DRM_ERROR("Relocation beyond object bounds: "
+                                  "obj %p target %d offset %d size %d.\n",
+                                  obj, reloc->target_handle,
+                                  (int) reloc->offset, (int) obj->size);
+                        drm_gem_object_unreference(target_obj);
+                        i915_gem_object_unpin(obj);
+                        return -EINVAL;
+                }
+                if (reloc->offset & 3) {
+                        DRM_ERROR("Relocation not 4-byte aligned: "
+                                  "obj %p target %d offset %d.\n",
+                                  obj, reloc->target_handle,
+                                  (int) reloc->offset);
+                        drm_gem_object_unreference(target_obj);
+                        i915_gem_object_unpin(obj);
+                        return -EINVAL;
+                }
+
+                /* and points to somewhere within the target object. */
+                if (reloc->delta >= target_obj->size) {
+                        DRM_ERROR("Relocation beyond target object bounds: "
+                                  "obj %p target %d delta %d size %d.\n",
+                                  obj, reloc->target_handle,
+                                  (int) reloc->delta, (int) target_obj->size);
+                        drm_gem_object_unreference(target_obj);
+                        i915_gem_object_unpin(obj);
+                        return -EINVAL;
+                }
+
                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
                 if (ret != 0) {
                         drm_gem_object_unreference(target_obj);
@@ -3126,6 +3352,8 @@ i915_dispatch_gem_execbuffer(struct drm_device *dev,
         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
         exec_len = (uint32_t) exec->batch_len;
 
+        trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno);
+
         count = nbox ? nbox : 1;
 
         for (i = 0; i < count; i++) {
@@ -3363,7 +3591,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
         i915_verify_inactive(dev, __FILE__, __LINE__);
 
-        if (dev_priv->mm.wedged) {
+        if (atomic_read(&dev_priv->mm.wedged)) {
                 DRM_ERROR("Execbuf while wedged\n");
                 mutex_unlock(&dev->struct_mutex);
                 ret = -EIO;
@@ -3421,8 +3649,23 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
                 /* error other than GTT full, or we've already tried again */
                 if (ret != -ENOSPC || pin_tries >= 1) {
-                        if (ret != -ERESTARTSYS)
-                                DRM_ERROR("Failed to pin buffers %d\n", ret);
+                        if (ret != -ERESTARTSYS) {
+                                unsigned long long total_size = 0;
+                                for (i = 0; i < args->buffer_count; i++)
+                                        total_size += object_list[i]->size;
+                                DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
+                                          pinned+1, args->buffer_count,
+                                          total_size, ret);
+                                DRM_ERROR("%d objects [%d pinned], "
+                                          "%d object bytes [%d pinned], "
+                                          "%d/%d gtt bytes\n",
+                                          atomic_read(&dev->object_count),
+                                          atomic_read(&dev->pin_count),
+                                          atomic_read(&dev->object_memory),
+                                          atomic_read(&dev->pin_memory),
+                                          atomic_read(&dev->gtt_memory),
+                                          dev->gtt_total);
+                        }
                         goto err;
                 }
 
@@ -3433,7 +3676,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
                 /* evict everyone we can from the aperture */
                 ret = i915_gem_evict_everything(dev);
-                if (ret)
+                if (ret && ret != -ENOSPC)
                         goto err;
         }
 
@@ -3489,8 +3732,12 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
         for (i = 0; i < args->buffer_count; i++) {
                 struct drm_gem_object *obj = object_list[i];
+                uint32_t old_write_domain = obj->write_domain;
 
                 obj->write_domain = obj->pending_write_domain;
+                trace_i915_gem_object_change_domain(obj,
+                                                    obj->read_domains,
+                                                    old_write_domain);
         }
 
         i915_verify_inactive(dev, __FILE__, __LINE__);
@@ -3607,11 +3854,8 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
         i915_verify_inactive(dev, __FILE__, __LINE__);
         if (obj_priv->gtt_space == NULL) {
                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
-                if (ret != 0) {
-                        if (ret != -EBUSY && ret != -ERESTARTSYS)
-                                DRM_ERROR("Failure to bind: %d\n", ret);
+                if (ret)
                         return ret;
-                }
         }
         /*
          * Pre-965 chips need a fence register set up in order to
@@ -3691,6 +3935,13 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
         }
         obj_priv = obj->driver_private;
 
+        if (obj_priv->madv != I915_MADV_WILLNEED) {
+                DRM_ERROR("Attempting to pin a purgeable buffer\n");
+                drm_gem_object_unreference(obj);
+                mutex_unlock(&dev->struct_mutex);
+                return -EINVAL;
+        }
+
         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
@@ -3803,6 +4054,56 @@ i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
     return i915_gem_ring_throttle(dev, file_priv);
 }
 
+int
+i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
+                       struct drm_file *file_priv)
+{
+        struct drm_i915_gem_madvise *args = data;
+        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj_priv;
+
+        switch (args->madv) {
+        case I915_MADV_DONTNEED:
+        case I915_MADV_WILLNEED:
+            break;
+        default:
+            return -EINVAL;
+        }
+
+        obj = drm_gem_object_lookup(dev, file_priv, args->handle);
+        if (obj == NULL) {
+                DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
+                          args->handle);
+                return -EBADF;
+        }
+
+        mutex_lock(&dev->struct_mutex);
+        obj_priv = obj->driver_private;
+
+        if (obj_priv->pin_count) {
+                drm_gem_object_unreference(obj);
+                mutex_unlock(&dev->struct_mutex);
+
+                DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
+                return -EINVAL;
+        }
+
+        if (obj_priv->madv != __I915_MADV_PURGED)
+                obj_priv->madv = args->madv;
+
+        /* if the object is no longer bound, discard its backing storage */
+        if (i915_gem_object_is_purgeable(obj_priv) &&
+            obj_priv->gtt_space == NULL)
+                i915_gem_object_truncate(obj);
+
+        args->retained = obj_priv->madv != __I915_MADV_PURGED;
+
+        drm_gem_object_unreference(obj);
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
 int i915_gem_init_object(struct drm_gem_object *obj)
 {
         struct drm_i915_gem_object *obj_priv;
@@ -3827,6 +4128,9 @@ int i915_gem_init_object(struct drm_gem_object *obj)
         obj_priv->fence_reg = I915_FENCE_REG_NONE;
         INIT_LIST_HEAD(&obj_priv->list);
         INIT_LIST_HEAD(&obj_priv->fence_list);
+        obj_priv->madv = I915_MADV_WILLNEED;
+
+        trace_i915_gem_object_create(obj);
 
         return 0;
 }
@@ -3836,6 +4140,8 @@ void i915_gem_free_object(struct drm_gem_object *obj)
         struct drm_device *dev = obj->dev;
         struct drm_i915_gem_object *obj_priv = obj->driver_private;
 
+        trace_i915_gem_object_destroy(obj);
+
         while (obj_priv->pin_count > 0)
                 i915_gem_object_unpin(obj);
 
@@ -3844,43 +4150,35 @@ void i915_gem_free_object(struct drm_gem_object *obj)
 
         i915_gem_object_unbind(obj);
 
-        i915_gem_free_mmap_offset(obj);
+        if (obj_priv->mmap_offset)
+                i915_gem_free_mmap_offset(obj);
 
         kfree(obj_priv->page_cpu_valid);
         kfree(obj_priv->bit_17);
         kfree(obj->driver_private);
 }
 
-/** Unbinds all objects that are on the given buffer list. */
+/** Unbinds all inactive objects. */
 static int
-i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
+i915_gem_evict_from_inactive_list(struct drm_device *dev)
 {
-        struct drm_gem_object *obj;
-        struct drm_i915_gem_object *obj_priv;
-        int ret;
+        drm_i915_private_t *dev_priv = dev->dev_private;
 
-        while (!list_empty(head)) {
-                obj_priv = list_first_entry(head,
-                                            struct drm_i915_gem_object,
-                                            list);
-                obj = obj_priv->obj;
+        while (!list_empty(&dev_priv->mm.inactive_list)) {
+                struct drm_gem_object *obj;
+                int ret;
 
-                if (obj_priv->pin_count != 0) {
-                        DRM_ERROR("Pinned object in unbind list\n");
-                        mutex_unlock(&dev->struct_mutex);
-                        return -EINVAL;
-                }
+                obj = list_first_entry(&dev_priv->mm.inactive_list,
+                                       struct drm_i915_gem_object,
+                                       list)->obj;
 
                 ret = i915_gem_object_unbind(obj);
                 if (ret != 0) {
-                        DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
-                                  ret);
-                        mutex_unlock(&dev->struct_mutex);
+                        DRM_ERROR("Error unbinding object: %d\n", ret);
                         return ret;
                 }
         }
 
-
         return 0;
 }
 
@@ -3902,6 +4200,7 @@ i915_gem_idle(struct drm_device *dev)
          * We need to replace this with a semaphore, or something.
          */
         dev_priv->mm.suspended = 1;
+        del_timer(&dev_priv->hangcheck_timer);
 
         /* Cancel the retire work handler, wait for it to finish if running
          */
@@ -3931,7 +4230,7 @@ i915_gem_idle(struct drm_device *dev)
                 if (last_seqno == cur_seqno) {
                         if (stuck++ > 100) {
                                 DRM_ERROR("hardware wedged\n");
-                                dev_priv->mm.wedged = 1;
+                                atomic_set(&dev_priv->mm.wedged, 1);
                                 DRM_WAKEUP(&dev_priv->irq_queue);
                                 break;
                         }
@@ -3944,7 +4243,7 @@ i915_gem_idle(struct drm_device *dev)
         i915_gem_retire_requests(dev);
 
         spin_lock(&dev_priv->mm.active_list_lock);
-        if (!dev_priv->mm.wedged) {
+        if (!atomic_read(&dev_priv->mm.wedged)) {
                 /* Active and flushing should now be empty as we've
                  * waited for a sequence higher than any pending execbuffer
                  */
@@ -3962,29 +4261,41 @@ i915_gem_idle(struct drm_device *dev)
          * the GPU domains and just stuff them onto inactive.
          */
         while (!list_empty(&dev_priv->mm.active_list)) {
-                struct drm_i915_gem_object *obj_priv;
+                struct drm_gem_object *obj;
+                uint32_t old_write_domain;
 
-                obj_priv = list_first_entry(&dev_priv->mm.active_list,
-                                            struct drm_i915_gem_object,
-                                            list);
-                obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
-                i915_gem_object_move_to_inactive(obj_priv->obj);
+                obj = list_first_entry(&dev_priv->mm.active_list,
+                                       struct drm_i915_gem_object,
+                                       list)->obj;
+                old_write_domain = obj->write_domain;
+                obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
+                i915_gem_object_move_to_inactive(obj);
+
+                trace_i915_gem_object_change_domain(obj,
+                                                    obj->read_domains,
+                                                    old_write_domain);
         }
         spin_unlock(&dev_priv->mm.active_list_lock);
 
         while (!list_empty(&dev_priv->mm.flushing_list)) {
-                struct drm_i915_gem_object *obj_priv;
+                struct drm_gem_object *obj;
+                uint32_t old_write_domain;
 
-                obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
-                                            struct drm_i915_gem_object,
-                                            list);
-                obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
-                i915_gem_object_move_to_inactive(obj_priv->obj);
+                obj = list_first_entry(&dev_priv->mm.flushing_list,
+                                       struct drm_i915_gem_object,
+                                       list)->obj;
+                old_write_domain = obj->write_domain;
+                obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
+                i915_gem_object_move_to_inactive(obj);
+
+                trace_i915_gem_object_change_domain(obj,
+                                                    obj->read_domains,
+                                                    old_write_domain);
         }
 
 
         /* Move all inactive buffers out of the GTT. */
-        ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
+        ret = i915_gem_evict_from_inactive_list(dev);
         WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
         if (ret) {
                 mutex_unlock(&dev->struct_mutex);
@@ -4206,9 +4517,9 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return 0;
 
-        if (dev_priv->mm.wedged) {
+        if (atomic_read(&dev_priv->mm.wedged)) {
                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
-                dev_priv->mm.wedged = 0;
+                atomic_set(&dev_priv->mm.wedged, 0);
         }
 
         mutex_lock(&dev->struct_mutex);
@@ -4274,6 +4585,10 @@ i915_gem_load(struct drm_device *dev)
                           i915_gem_retire_work_handler);
         dev_priv->mm.next_gem_seqno = 1;
 
+        spin_lock(&shrink_list_lock);
+        list_add(&dev_priv->mm.shrink_list, &shrink_list);
+        spin_unlock(&shrink_list_lock);
+
         /* Old X drivers will take 0-2 for front, back, depth buffers */
         dev_priv->fence_reg_start = 3;
 
@@ -4491,3 +4806,116 @@ void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
                 list_del_init(i915_file_priv->mm.request_list.next);
         mutex_unlock(&dev->struct_mutex);
 }
+
+static int
+i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
+{
+        drm_i915_private_t *dev_priv, *next_dev;
+        struct drm_i915_gem_object *obj_priv, *next_obj;
+        int cnt = 0;
+        int would_deadlock = 1;
+
+        /* "fast-path" to count number of available objects */
+        if (nr_to_scan == 0) {
+                spin_lock(&shrink_list_lock);
+                list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
+                        struct drm_device *dev = dev_priv->dev;
+
+                        if (mutex_trylock(&dev->struct_mutex)) {
+                                list_for_each_entry(obj_priv,
+                                                    &dev_priv->mm.inactive_list,
+                                                    list)
+                                        cnt++;
+                                mutex_unlock(&dev->struct_mutex);
+                        }
+                }
+                spin_unlock(&shrink_list_lock);
+
+                return (cnt / 100) * sysctl_vfs_cache_pressure;
+        }
+
+        spin_lock(&shrink_list_lock);
+
+        /* first scan for clean buffers */
+        list_for_each_entry_safe(dev_priv, next_dev,
+                                 &shrink_list, mm.shrink_list) {
+                struct drm_device *dev = dev_priv->dev;
+
+                if (! mutex_trylock(&dev->struct_mutex))
+                        continue;
+
+                spin_unlock(&shrink_list_lock);
+
+                i915_gem_retire_requests(dev);
+
+                list_for_each_entry_safe(obj_priv, next_obj,
+                                         &dev_priv->mm.inactive_list,
+                                         list) {
+                        if (i915_gem_object_is_purgeable(obj_priv)) {
+                                i915_gem_object_unbind(obj_priv->obj);
+                                if (--nr_to_scan <= 0)
+                                        break;
+                        }
+                }
+
+                spin_lock(&shrink_list_lock);
+                mutex_unlock(&dev->struct_mutex);
+
+                would_deadlock = 0;
+
+                if (nr_to_scan <= 0)
+                        break;
+        }
+
+        /* second pass, evict/count anything still on the inactive list */
+        list_for_each_entry_safe(dev_priv, next_dev,
+                                 &shrink_list, mm.shrink_list) {
+                struct drm_device *dev = dev_priv->dev;
+
+                if (! mutex_trylock(&dev->struct_mutex))
+                        continue;
+
+                spin_unlock(&shrink_list_lock);
+
+                list_for_each_entry_safe(obj_priv, next_obj,
+                                         &dev_priv->mm.inactive_list,
+                                         list) {
+                        if (nr_to_scan > 0) {
+                                i915_gem_object_unbind(obj_priv->obj);
+                                nr_to_scan--;
+                        } else
+                                cnt++;
+                }
+
+                spin_lock(&shrink_list_lock);
+                mutex_unlock(&dev->struct_mutex);
+
+                would_deadlock = 0;
+        }
+
+        spin_unlock(&shrink_list_lock);
+
+        if (would_deadlock)
+                return -1;
+        else if (cnt > 0)
+                return (cnt / 100) * sysctl_vfs_cache_pressure;
+        else
+                return 0;
+}
+
+static struct shrinker shrinker = {
+        .shrink = i915_gem_shrink,
+        .seeks = DEFAULT_SEEKS,
+};
+
+__init void
+i915_gem_shrinker_init(void)
+{
+    register_shrinker(&shrinker);
+}
+
+__exit void
+i915_gem_shrinker_exit(void)
+{
+    unregister_shrinker(&shrinker);
+}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 6c89f2ff2495..4dfeec7cdd42 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -31,6 +31,7 @@
 #include "drm.h"
 #include "i915_drm.h"
 #include "i915_drv.h"
+#include "i915_trace.h"
 #include "intel_drv.h"
 
 #define MAX_NOPID ((u32)~0)
@@ -279,7 +280,9 @@ irqreturn_t igdng_irq_handler(struct drm_device *dev)
                 }
 
                 if (gt_iir & GT_USER_INTERRUPT) {
-                        dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
+                        u32 seqno = i915_get_gem_seqno(dev);
+                        dev_priv->mm.irq_gem_seqno = seqno;
+                        trace_i915_gem_request_complete(dev, seqno);
                         DRM_WAKEUP(&dev_priv->irq_queue);
                 }
 
@@ -302,12 +305,25 @@ static void i915_error_work_func(struct work_struct *work)
         drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
                                                     error_work);
         struct drm_device *dev = dev_priv->dev;
-        char *event_string = "ERROR=1";
-        char *envp[] = { event_string, NULL };
+        char *error_event[] = { "ERROR=1", NULL };
+        char *reset_event[] = { "RESET=1", NULL };
+        char *reset_done_event[] = { "ERROR=0", NULL };
 
         DRM_DEBUG("generating error event\n");
-
-        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, envp);
+        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
+
+        if (atomic_read(&dev_priv->mm.wedged)) {
+                if (IS_I965G(dev)) {
+                        DRM_DEBUG("resetting chip\n");
+                        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event);
+                        if (!i965_reset(dev, GDRST_RENDER)) {
+                                atomic_set(&dev_priv->mm.wedged, 0);
+                                kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event);
+                        }
+                } else {
+                        printk("reboot required\n");
+                }
+        }
 }
 
 /**
@@ -372,7 +388,7 @@ out:
  * so userspace knows something bad happened (should trigger collection
  * of a ring dump etc.).
  */
-static void i915_handle_error(struct drm_device *dev)
+static void i915_handle_error(struct drm_device *dev, bool wedged)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 eir = I915_READ(EIR);
@@ -482,6 +498,16 @@ static void i915_handle_error(struct drm_device *dev)
                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
         }
 
+        if (wedged) {
+                atomic_set(&dev_priv->mm.wedged, 1);
+
+                /*
+                 * Wakeup waiting processes so they don't hang
+                 */
+                printk("i915: Waking up sleeping processes\n");
+                DRM_WAKEUP(&dev_priv->irq_queue);
+        }
+
         queue_work(dev_priv->wq, &dev_priv->error_work);
 }
 
@@ -527,7 +553,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                 pipeb_stats = I915_READ(PIPEBSTAT);
 
                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        i915_handle_error(dev);
+                        i915_handle_error(dev, false);
 
                 /*
                  * Clear the PIPE(A|B)STAT regs before the IIR
@@ -599,8 +625,12 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
                 }
 
                 if (iir & I915_USER_INTERRUPT) {
-                        dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
+                        u32 seqno = i915_get_gem_seqno(dev);
+                        dev_priv->mm.irq_gem_seqno = seqno;
+                        trace_i915_gem_request_complete(dev, seqno);
                         DRM_WAKEUP(&dev_priv->irq_queue);
+                        dev_priv->hangcheck_count = 0;
+                        mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
                 }
 
                 if (pipea_stats & vblank_status) {
@@ -880,6 +910,52 @@ int i915_vblank_swap(struct drm_device *dev, void *data,
         return -EINVAL;
 }
 
+struct drm_i915_gem_request *i915_get_tail_request(struct drm_device *dev) {
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        return list_entry(dev_priv->mm.request_list.prev, struct drm_i915_gem_request, list);
+}
+
+/**
+ * This is called when the chip hasn't reported back with completed
+ * batchbuffers in a long time. The first time this is called we simply record
+ * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses
+ * again, we assume the chip is wedged and try to fix it.
+ */
+void i915_hangcheck_elapsed(unsigned long data)
+{
+        struct drm_device *dev = (struct drm_device *)data;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        uint32_t acthd;
+       
+        if (!IS_I965G(dev))
+                acthd = I915_READ(ACTHD);
+        else
+                acthd = I915_READ(ACTHD_I965);
+
+        /* If all work is done then ACTHD clearly hasn't advanced. */
+        if (list_empty(&dev_priv->mm.request_list) ||
+                       i915_seqno_passed(i915_get_gem_seqno(dev), i915_get_tail_request(dev)->seqno)) {
+                dev_priv->hangcheck_count = 0;
+                return;
+        }
+
+        if (dev_priv->last_acthd == acthd && dev_priv->hangcheck_count > 0) {
+                DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
+                i915_handle_error(dev, true);
+                return;
+        } 
+
+        /* Reset timer case chip hangs without another request being added */
+        mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
+
+        if (acthd != dev_priv->last_acthd)
+                dev_priv->hangcheck_count = 0;
+        else
+                dev_priv->hangcheck_count++;
+
+        dev_priv->last_acthd = acthd;
+}
+
 /* drm_dma.h hooks
 */
 static void igdng_irq_preinstall(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/i915_opregion.c b/drivers/gpu/drm/i915/i915_opregion.c
index e4b4e8898e39..2d5193556d3f 100644
--- a/drivers/gpu/drm/i915/i915_opregion.c
+++ b/drivers/gpu/drm/i915/i915_opregion.c
@@ -148,6 +148,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_asle *asle = dev_priv->opregion.asle;
         u32 blc_pwm_ctl, blc_pwm_ctl2;
+        u32 max_backlight, level, shift;
 
         if (!(bclp & ASLE_BCLP_VALID))
                 return ASLE_BACKLIGHT_FAIL;
@@ -157,14 +158,25 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
                 return ASLE_BACKLIGHT_FAIL;
 
         blc_pwm_ctl = I915_READ(BLC_PWM_CTL);
-        blc_pwm_ctl &= ~BACKLIGHT_DUTY_CYCLE_MASK;
         blc_pwm_ctl2 = I915_READ(BLC_PWM_CTL2);
 
-        if (blc_pwm_ctl2 & BLM_COMBINATION_MODE)
+        if (IS_I965G(dev) && (blc_pwm_ctl2 & BLM_COMBINATION_MODE))
                 pci_write_config_dword(dev->pdev, PCI_LBPC, bclp);
-        else
-                I915_WRITE(BLC_PWM_CTL, blc_pwm_ctl | ((bclp * 0x101)-1));
-
+        else {
+                if (IS_IGD(dev)) {
+                        blc_pwm_ctl &= ~(BACKLIGHT_DUTY_CYCLE_MASK - 1);
+                        max_backlight = (blc_pwm_ctl & BACKLIGHT_MODULATION_FREQ_MASK) >> 
+                                        BACKLIGHT_MODULATION_FREQ_SHIFT;
+                        shift = BACKLIGHT_DUTY_CYCLE_SHIFT + 1;
+                } else {
+                        blc_pwm_ctl &= ~BACKLIGHT_DUTY_CYCLE_MASK;
+                        max_backlight = ((blc_pwm_ctl & BACKLIGHT_MODULATION_FREQ_MASK) >> 
+                                        BACKLIGHT_MODULATION_FREQ_SHIFT) * 2;
+                        shift = BACKLIGHT_DUTY_CYCLE_SHIFT;
+                }
+                level = (bclp * max_backlight) / 255;
+                I915_WRITE(BLC_PWM_CTL, blc_pwm_ctl | (level << shift));
+        }
         asle->cblv = (bclp*0x64)/0xff | ASLE_CBLV_VALID;
 
         return 0;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 3f7963553464..0466ddbeba32 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -86,6 +86,10 @@
 #define   I915_GC_RENDER_CLOCK_200_MHZ  (1 << 0)
 #define   I915_GC_RENDER_CLOCK_333_MHZ  (4 << 0)
 #define LBB     0xf4
+#define GDRST 0xc0
+#define  GDRST_FULL     (0<<2)
+#define  GDRST_RENDER   (1<<2)
+#define  GDRST_MEDIA    (3<<2)
 
 /* VGA stuff */
 
@@ -344,9 +348,37 @@
 #define   FBC_CTL_PLANEA        (0<<0)
 #define   FBC_CTL_PLANEB        (1<<0)
 #define FBC_FENCE_OFF           0x0321b
+#define FBC_TAG                 0x03300
 
 #define FBC_LL_SIZE             (1536)
 
+/* Framebuffer compression for GM45+ */
+#define DPFC_CB_BASE            0x3200
+#define DPFC_CONTROL            0x3208
+#define   DPFC_CTL_EN           (1<<31)
+#define   DPFC_CTL_PLANEA       (0<<30)
+#define   DPFC_CTL_PLANEB       (1<<30)
+#define   DPFC_CTL_FENCE_EN     (1<<29)
+#define   DPFC_SR_EN            (1<<10)
+#define   DPFC_CTL_LIMIT_1X     (0<<6)
+#define   DPFC_CTL_LIMIT_2X     (1<<6)
+#define   DPFC_CTL_LIMIT_4X     (2<<6)
+#define DPFC_RECOMP_CTL         0x320c
+#define   DPFC_RECOMP_STALL_EN  (1<<27)
+#define   DPFC_RECOMP_STALL_WM_SHIFT (16)
+#define   DPFC_RECOMP_STALL_WM_MASK (0x07ff0000)
+#define   DPFC_RECOMP_TIMER_COUNT_SHIFT (0)
+#define   DPFC_RECOMP_TIMER_COUNT_MASK (0x0000003f)
+#define DPFC_STATUS             0x3210
+#define   DPFC_INVAL_SEG_SHIFT  (16)
+#define   DPFC_INVAL_SEG_MASK   (0x07ff0000)
+#define   DPFC_COMP_SEG_SHIFT   (0)
+#define   DPFC_COMP_SEG_MASK    (0x000003ff)
+#define DPFC_STATUS2            0x3214
+#define DPFC_FENCE_YOFF         0x3218
+#define DPFC_CHICKEN            0x3224
+#define   DPFC_HT_MODIFY        (1<<31)
+
 /*
  * GPIO regs
  */
@@ -2000,6 +2032,8 @@
 #define  PF_ENABLE              (1<<31)
 #define PFA_WIN_SZ              0x68074
 #define PFB_WIN_SZ              0x68874
+#define PFA_WIN_POS             0x68070
+#define PFB_WIN_POS             0x68870
 
 /* legacy palette */
 #define LGC_PALETTE_A           0x4a000
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 20d4d19f5568..bd6d8d91ca9f 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -228,6 +228,7 @@ static void i915_save_modeset_reg(struct drm_device *dev)
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return;
+
         /* Pipe & plane A info */
         dev_priv->savePIPEACONF = I915_READ(PIPEACONF);
         dev_priv->savePIPEASRC = I915_READ(PIPEASRC);
@@ -285,6 +286,7 @@ static void i915_save_modeset_reg(struct drm_device *dev)
         dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
         return;
 }
+
 static void i915_restore_modeset_reg(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -379,19 +381,10 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
 
         return;
 }
-int i915_save_state(struct drm_device *dev)
+
+void i915_save_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int i;
-
-        pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB);
-
-        /* Render Standby */
-        if (IS_I965G(dev) && IS_MOBILE(dev))
-                dev_priv->saveRENDERSTANDBY = I915_READ(MCHBAR_RENDER_STANDBY);
-
-        /* Hardware status page */
-        dev_priv->saveHWS = I915_READ(HWS_PGA);
 
         /* Display arbitration control */
         dev_priv->saveDSPARB = I915_READ(DSPARB);
@@ -399,6 +392,7 @@ int i915_save_state(struct drm_device *dev)
         /* This is only meaningful in non-KMS mode */
         /* Don't save them in KMS mode */
         i915_save_modeset_reg(dev);
+
         /* Cursor state */
         dev_priv->saveCURACNTR = I915_READ(CURACNTR);
         dev_priv->saveCURAPOS = I915_READ(CURAPOS);
@@ -448,81 +442,22 @@ int i915_save_state(struct drm_device *dev)
         dev_priv->saveFBC_CONTROL2 = I915_READ(FBC_CONTROL2);
         dev_priv->saveFBC_CONTROL = I915_READ(FBC_CONTROL);
 
-        /* Interrupt state */
-        dev_priv->saveIIR = I915_READ(IIR);
-        dev_priv->saveIER = I915_READ(IER);
-        dev_priv->saveIMR = I915_READ(IMR);
-
         /* VGA state */
         dev_priv->saveVGA0 = I915_READ(VGA0);
         dev_priv->saveVGA1 = I915_READ(VGA1);
         dev_priv->saveVGA_PD = I915_READ(VGA_PD);
         dev_priv->saveVGACNTRL = I915_READ(VGACNTRL);
 
-        /* Clock gating state */
-        dev_priv->saveD_STATE = I915_READ(D_STATE);
-        dev_priv->saveDSPCLK_GATE_D = I915_READ(DSPCLK_GATE_D);
-
-        /* Cache mode state */
-        dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
-
-        /* Memory Arbitration state */
-        dev_priv->saveMI_ARB_STATE = I915_READ(MI_ARB_STATE);
-
-        /* Scratch space */
-        for (i = 0; i < 16; i++) {
-                dev_priv->saveSWF0[i] = I915_READ(SWF00 + (i << 2));
-                dev_priv->saveSWF1[i] = I915_READ(SWF10 + (i << 2));
-        }
-        for (i = 0; i < 3; i++)
-                dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
-
-        /* Fences */
-        if (IS_I965G(dev)) {
-                for (i = 0; i < 16; i++)
-                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
-        } else {
-                for (i = 0; i < 8; i++)
-                        dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
-
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
-        }
         i915_save_vga(dev);
-
-        return 0;
 }
 
-int i915_restore_state(struct drm_device *dev)
+void i915_restore_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int i;
-
-        pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB);
-
-        /* Render Standby */
-        if (IS_I965G(dev) && IS_MOBILE(dev))
-                I915_WRITE(MCHBAR_RENDER_STANDBY, dev_priv->saveRENDERSTANDBY);
-
-        /* Hardware status page */
-        I915_WRITE(HWS_PGA, dev_priv->saveHWS);
 
         /* Display arbitration */
         I915_WRITE(DSPARB, dev_priv->saveDSPARB);
 
-        /* Fences */
-        if (IS_I965G(dev)) {
-                for (i = 0; i < 16; i++)
-                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
-        } else {
-                for (i = 0; i < 8; i++)
-                        I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
-                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-                        for (i = 0; i < 8; i++)
-                                I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
-        }
-        
         /* Display port ratios (must be done before clock is set) */
         if (SUPPORTS_INTEGRATED_DP(dev)) {
                 I915_WRITE(PIPEA_GMCH_DATA_M, dev_priv->savePIPEA_GMCH_DATA_M);
@@ -534,9 +469,11 @@ int i915_restore_state(struct drm_device *dev)
                 I915_WRITE(PIPEA_DP_LINK_N, dev_priv->savePIPEA_DP_LINK_N);
                 I915_WRITE(PIPEB_DP_LINK_N, dev_priv->savePIPEB_DP_LINK_N);
         }
+
         /* This is only meaningful in non-KMS mode */
         /* Don't restore them in KMS mode */
         i915_restore_modeset_reg(dev);
+
         /* Cursor state */
         I915_WRITE(CURAPOS, dev_priv->saveCURAPOS);
         I915_WRITE(CURACNTR, dev_priv->saveCURACNTR);
@@ -586,6 +523,95 @@ int i915_restore_state(struct drm_device *dev)
         I915_WRITE(VGA_PD, dev_priv->saveVGA_PD);
         DRM_UDELAY(150);
 
+        i915_restore_vga(dev);
+}
+
+int i915_save_state(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
+
+        pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB);
+
+        /* Render Standby */
+        if (IS_I965G(dev) && IS_MOBILE(dev))
+                dev_priv->saveRENDERSTANDBY = I915_READ(MCHBAR_RENDER_STANDBY);
+
+        /* Hardware status page */
+        dev_priv->saveHWS = I915_READ(HWS_PGA);
+
+        i915_save_display(dev);
+
+        /* Interrupt state */
+        dev_priv->saveIER = I915_READ(IER);
+        dev_priv->saveIMR = I915_READ(IMR);
+
+        /* Clock gating state */
+        dev_priv->saveD_STATE = I915_READ(D_STATE);
+        dev_priv->saveDSPCLK_GATE_D = I915_READ(DSPCLK_GATE_D); /* Not sure about this */
+
+        /* Cache mode state */
+        dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
+
+        /* Memory Arbitration state */
+        dev_priv->saveMI_ARB_STATE = I915_READ(MI_ARB_STATE);
+
+        /* Scratch space */
+        for (i = 0; i < 16; i++) {
+                dev_priv->saveSWF0[i] = I915_READ(SWF00 + (i << 2));
+                dev_priv->saveSWF1[i] = I915_READ(SWF10 + (i << 2));
+        }
+        for (i = 0; i < 3; i++)
+                dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
+
+        /* Fences */
+        if (IS_I965G(dev)) {
+                for (i = 0; i < 16; i++)
+                        dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
+        } else {
+                for (i = 0; i < 8; i++)
+                        dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
+
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
+        }
+
+        return 0;
+}
+
+int i915_restore_state(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int i;
+
+        pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB);
+
+        /* Render Standby */
+        if (IS_I965G(dev) && IS_MOBILE(dev))
+                I915_WRITE(MCHBAR_RENDER_STANDBY, dev_priv->saveRENDERSTANDBY);
+
+        /* Hardware status page */
+        I915_WRITE(HWS_PGA, dev_priv->saveHWS);
+
+        /* Fences */
+        if (IS_I965G(dev)) {
+                for (i = 0; i < 16; i++)
+                        I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
+        } else {
+                for (i = 0; i < 8; i++)
+                        I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
+                if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+                        for (i = 0; i < 8; i++)
+                                I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
+        }
+
+        i915_restore_display(dev);
+
+        /* Interrupt state */
+        I915_WRITE (IER, dev_priv->saveIER);
+        I915_WRITE (IMR,  dev_priv->saveIMR);
+
         /* Clock gating state */
         I915_WRITE (D_STATE, dev_priv->saveD_STATE);
         I915_WRITE (DSPCLK_GATE_D, dev_priv->saveDSPCLK_GATE_D);
@@ -603,8 +629,6 @@ int i915_restore_state(struct drm_device *dev)
         for (i = 0; i < 3; i++)
                 I915_WRITE(SWF30 + (i << 2), dev_priv->saveSWF2[i]);
 
-        i915_restore_vga(dev);
-
         return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
new file mode 100644
index 000000000000..5567a40816f3
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -0,0 +1,315 @@
+#if !defined(_I915_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _I915_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include <drm/drmP.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM i915
+#define TRACE_SYSTEM_STRING __stringify(TRACE_SYSTEM)
+#define TRACE_INCLUDE_FILE i915_trace
+
+/* object tracking */
+
+TRACE_EVENT(i915_gem_object_create,
+
+            TP_PROTO(struct drm_gem_object *obj),
+
+            TP_ARGS(obj),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             __field(u32, size)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->size = obj->size;
+                           ),
+
+            TP_printk("obj=%p, size=%u", __entry->obj, __entry->size)
+);
+
+TRACE_EVENT(i915_gem_object_bind,
+
+            TP_PROTO(struct drm_gem_object *obj, u32 gtt_offset),
+
+            TP_ARGS(obj, gtt_offset),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             __field(u32, gtt_offset)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->gtt_offset = gtt_offset;
+                           ),
+
+            TP_printk("obj=%p, gtt_offset=%08x",
+                      __entry->obj, __entry->gtt_offset)
+);
+
+TRACE_EVENT(i915_gem_object_clflush,
+
+            TP_PROTO(struct drm_gem_object *obj),
+
+            TP_ARGS(obj),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           ),
+
+            TP_printk("obj=%p", __entry->obj)
+);
+
+TRACE_EVENT(i915_gem_object_change_domain,
+
+            TP_PROTO(struct drm_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain),
+
+            TP_ARGS(obj, old_read_domains, old_write_domain),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             __field(u32, read_domains)
+                             __field(u32, write_domain)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->read_domains = obj->read_domains | (old_read_domains << 16);
+                           __entry->write_domain = obj->write_domain | (old_write_domain << 16);
+                           ),
+
+            TP_printk("obj=%p, read=%04x, write=%04x",
+                      __entry->obj,
+                      __entry->read_domains, __entry->write_domain)
+);
+
+TRACE_EVENT(i915_gem_object_get_fence,
+
+            TP_PROTO(struct drm_gem_object *obj, int fence, int tiling_mode),
+
+            TP_ARGS(obj, fence, tiling_mode),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             __field(int, fence)
+                             __field(int, tiling_mode)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           __entry->fence = fence;
+                           __entry->tiling_mode = tiling_mode;
+                           ),
+
+            TP_printk("obj=%p, fence=%d, tiling=%d",
+                      __entry->obj, __entry->fence, __entry->tiling_mode)
+);
+
+TRACE_EVENT(i915_gem_object_unbind,
+
+            TP_PROTO(struct drm_gem_object *obj),
+
+            TP_ARGS(obj),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           ),
+
+            TP_printk("obj=%p", __entry->obj)
+);
+
+TRACE_EVENT(i915_gem_object_destroy,
+
+            TP_PROTO(struct drm_gem_object *obj),
+
+            TP_ARGS(obj),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_gem_object *, obj)
+                             ),
+
+            TP_fast_assign(
+                           __entry->obj = obj;
+                           ),
+
+            TP_printk("obj=%p", __entry->obj)
+);
+
+/* batch tracing */
+
+TRACE_EVENT(i915_gem_request_submit,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             __field(u32, seqno)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           __entry->seqno = seqno;
+                           ),
+
+            TP_printk("dev=%p, seqno=%u", __entry->dev, __entry->seqno)
+);
+
+TRACE_EVENT(i915_gem_request_flush,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno,
+                     u32 flush_domains, u32 invalidate_domains),
+
+            TP_ARGS(dev, seqno, flush_domains, invalidate_domains),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             __field(u32, seqno)
+                             __field(u32, flush_domains)
+                             __field(u32, invalidate_domains)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           __entry->seqno = seqno;
+                           __entry->flush_domains = flush_domains;
+                           __entry->invalidate_domains = invalidate_domains;
+                           ),
+
+            TP_printk("dev=%p, seqno=%u, flush=%04x, invalidate=%04x",
+                      __entry->dev, __entry->seqno,
+                      __entry->flush_domains, __entry->invalidate_domains)
+);
+
+
+TRACE_EVENT(i915_gem_request_complete,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             __field(u32, seqno)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           __entry->seqno = seqno;
+                           ),
+
+            TP_printk("dev=%p, seqno=%u", __entry->dev, __entry->seqno)
+);
+
+TRACE_EVENT(i915_gem_request_retire,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             __field(u32, seqno)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           __entry->seqno = seqno;
+                           ),
+
+            TP_printk("dev=%p, seqno=%u", __entry->dev, __entry->seqno)
+);
+
+TRACE_EVENT(i915_gem_request_wait_begin,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             __field(u32, seqno)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           __entry->seqno = seqno;
+                           ),
+
+            TP_printk("dev=%p, seqno=%u", __entry->dev, __entry->seqno)
+);
+
+TRACE_EVENT(i915_gem_request_wait_end,
+
+            TP_PROTO(struct drm_device *dev, u32 seqno),
+
+            TP_ARGS(dev, seqno),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             __field(u32, seqno)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           __entry->seqno = seqno;
+                           ),
+
+            TP_printk("dev=%p, seqno=%u", __entry->dev, __entry->seqno)
+);
+
+TRACE_EVENT(i915_ring_wait_begin,
+
+            TP_PROTO(struct drm_device *dev),
+
+            TP_ARGS(dev),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           ),
+
+            TP_printk("dev=%p", __entry->dev)
+);
+
+TRACE_EVENT(i915_ring_wait_end,
+
+            TP_PROTO(struct drm_device *dev),
+
+            TP_ARGS(dev),
+
+            TP_STRUCT__entry(
+                             __field(struct drm_device *, dev)
+                             ),
+
+            TP_fast_assign(
+                           __entry->dev = dev;
+                           ),
+
+            TP_printk("dev=%p", __entry->dev)
+);
+
+#endif /* _I915_TRACE_H_ */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/i915
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/i915/i915_trace_points.c b/drivers/gpu/drm/i915/i915_trace_points.c
new file mode 100644
index 000000000000..ead876eb6ea0
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_trace_points.c
@@ -0,0 +1,11 @@
+/*
+ * Copyright © 2009 Intel Corporation
+ *
+ * Authors:
+ *    Chris Wilson <chris@chris-wilson.co.uk>
+ */
+
+#include "i915_drv.h"
+
+#define CREATE_TRACE_POINTS
+#include "i915_trace.h"
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 1e28c1652fd0..4337414846b6 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -217,6 +217,9 @@ parse_general_features(struct drm_i915_private *dev_priv,
                         if (IS_I85X(dev_priv->dev))
                                 dev_priv->lvds_ssc_freq =
                                         general->ssc_freq ? 66 : 48;
+                        else if (IS_IGDNG(dev_priv->dev))
+                                dev_priv->lvds_ssc_freq =
+                                        general->ssc_freq ? 100 : 120;
                         else
                                 dev_priv->lvds_ssc_freq =
                                         general->ssc_freq ? 100 : 96;
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 88814fa2dfd2..212e22740fc1 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -179,13 +179,10 @@ static bool intel_igdng_crt_detect_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 adpa, temp;
+        u32 adpa;
         bool ret;
 
-        temp = adpa = I915_READ(PCH_ADPA);
-
-        adpa &= ~ADPA_DAC_ENABLE;
-        I915_WRITE(PCH_ADPA, adpa);
+        adpa = I915_READ(PCH_ADPA);
 
         adpa &= ~ADPA_CRT_HOTPLUG_MASK;
 
@@ -212,8 +209,6 @@ static bool intel_igdng_crt_detect_hotplug(struct drm_connector *connector)
         else
                 ret = false;
 
-        /* restore origin register */
-        I915_WRITE(PCH_ADPA, temp);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 0227b1652906..93ff6c03733e 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -24,6 +24,8 @@
  *      Eric Anholt <eric@anholt.net>
  */
 
+#include <linux/module.h>
+#include <linux/input.h>
 #include <linux/i2c.h>
 #include <linux/kernel.h>
 #include "drmP.h"
@@ -875,7 +877,7 @@ intel_igdng_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
                                                refclk, best_clock);
 
         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-                if ((I915_READ(LVDS) & LVDS_CLKB_POWER_MASK) ==
+                if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) ==
                     LVDS_CLKB_POWER_UP)
                         clock.p2 = limit->p2.p2_fast;
                 else
@@ -952,6 +954,241 @@ intel_wait_for_vblank(struct drm_device *dev)
         mdelay(20);
 }
 
+/* Parameters have changed, update FBC info */
+static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_framebuffer *fb = crtc->fb;
+        struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+        struct drm_i915_gem_object *obj_priv = intel_fb->obj->driver_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int plane, i;
+        u32 fbc_ctl, fbc_ctl2;
+
+        dev_priv->cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
+
+        if (fb->pitch < dev_priv->cfb_pitch)
+                dev_priv->cfb_pitch = fb->pitch;
+
+        /* FBC_CTL wants 64B units */
+        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
+        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_plane = intel_crtc->plane;
+        plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
+
+        /* Clear old tags */
+        for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
+                I915_WRITE(FBC_TAG + (i * 4), 0);
+
+        /* Set it up... */
+        fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
+        if (obj_priv->tiling_mode != I915_TILING_NONE)
+                fbc_ctl2 |= FBC_CTL_CPU_FENCE;
+        I915_WRITE(FBC_CONTROL2, fbc_ctl2);
+        I915_WRITE(FBC_FENCE_OFF, crtc->y);
+
+        /* enable it... */
+        fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
+        fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
+        fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
+        if (obj_priv->tiling_mode != I915_TILING_NONE)
+                fbc_ctl |= dev_priv->cfb_fence;
+        I915_WRITE(FBC_CONTROL, fbc_ctl);
+
+        DRM_DEBUG("enabled FBC, pitch %ld, yoff %d, plane %d, ",
+                  dev_priv->cfb_pitch, crtc->y, dev_priv->cfb_plane);
+}
+
+void i8xx_disable_fbc(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 fbc_ctl;
+
+        if (!I915_HAS_FBC(dev))
+                return;
+
+        /* Disable compression */
+        fbc_ctl = I915_READ(FBC_CONTROL);
+        fbc_ctl &= ~FBC_CTL_EN;
+        I915_WRITE(FBC_CONTROL, fbc_ctl);
+
+        /* Wait for compressing bit to clear */
+        while (I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING)
+                ; /* nothing */
+
+        intel_wait_for_vblank(dev);
+
+        DRM_DEBUG("disabled FBC\n");
+}
+
+static bool i8xx_fbc_enabled(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
+}
+
+static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_framebuffer *fb = crtc->fb;
+        struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+        struct drm_i915_gem_object *obj_priv = intel_fb->obj->driver_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int plane = (intel_crtc->plane == 0 ? DPFC_CTL_PLANEA :
+                     DPFC_CTL_PLANEB);
+        unsigned long stall_watermark = 200;
+        u32 dpfc_ctl;
+
+        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
+        dev_priv->cfb_fence = obj_priv->fence_reg;
+        dev_priv->cfb_plane = intel_crtc->plane;
+
+        dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
+        if (obj_priv->tiling_mode != I915_TILING_NONE) {
+                dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
+                I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
+        } else {
+                I915_WRITE(DPFC_CHICKEN, ~DPFC_HT_MODIFY);
+        }
+
+        I915_WRITE(DPFC_CONTROL, dpfc_ctl);
+        I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
+                   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
+                   (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
+        I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
+
+        /* enable it... */
+        I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
+
+        DRM_DEBUG("enabled fbc on plane %d\n", intel_crtc->plane);
+}
+
+void g4x_disable_fbc(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 dpfc_ctl;
+
+        /* Disable compression */
+        dpfc_ctl = I915_READ(DPFC_CONTROL);
+        dpfc_ctl &= ~DPFC_CTL_EN;
+        I915_WRITE(DPFC_CONTROL, dpfc_ctl);
+        intel_wait_for_vblank(dev);
+
+        DRM_DEBUG("disabled FBC\n");
+}
+
+static bool g4x_fbc_enabled(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
+}
+
+/**
+ * intel_update_fbc - enable/disable FBC as needed
+ * @crtc: CRTC to point the compressor at
+ * @mode: mode in use
+ *
+ * Set up the framebuffer compression hardware at mode set time.  We
+ * enable it if possible:
+ *   - plane A only (on pre-965)
+ *   - no pixel mulitply/line duplication
+ *   - no alpha buffer discard
+ *   - no dual wide
+ *   - framebuffer <= 2048 in width, 1536 in height
+ *
+ * We can't assume that any compression will take place (worst case),
+ * so the compressed buffer has to be the same size as the uncompressed
+ * one.  It also must reside (along with the line length buffer) in
+ * stolen memory.
+ *
+ * We need to enable/disable FBC on a global basis.
+ */
+static void intel_update_fbc(struct drm_crtc *crtc,
+                             struct drm_display_mode *mode)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_framebuffer *fb = crtc->fb;
+        struct intel_framebuffer *intel_fb;
+        struct drm_i915_gem_object *obj_priv;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int plane = intel_crtc->plane;
+
+        if (!i915_powersave)
+                return;
+
+        if (!dev_priv->display.fbc_enabled ||
+            !dev_priv->display.enable_fbc ||
+            !dev_priv->display.disable_fbc)
+                return;
+
+        if (!crtc->fb)
+                return;
+
+        intel_fb = to_intel_framebuffer(fb);
+        obj_priv = intel_fb->obj->driver_private;
+
+        /*
+         * If FBC is already on, we just have to verify that we can
+         * keep it that way...
+         * Need to disable if:
+         *   - changing FBC params (stride, fence, mode)
+         *   - new fb is too large to fit in compressed buffer
+         *   - going to an unsupported config (interlace, pixel multiply, etc.)
+         */
+        if (intel_fb->obj->size > dev_priv->cfb_size) {
+                DRM_DEBUG("framebuffer too large, disabling compression\n");
+                goto out_disable;
+        }
+        if ((mode->flags & DRM_MODE_FLAG_INTERLACE) ||
+            (mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
+                DRM_DEBUG("mode incompatible with compression, disabling\n");
+                goto out_disable;
+        }
+        if ((mode->hdisplay > 2048) ||
+            (mode->vdisplay > 1536)) {
+                DRM_DEBUG("mode too large for compression, disabling\n");
+                goto out_disable;
+        }
+        if ((IS_I915GM(dev) || IS_I945GM(dev)) && plane != 0) {
+                DRM_DEBUG("plane not 0, disabling compression\n");
+                goto out_disable;
+        }
+        if (obj_priv->tiling_mode != I915_TILING_X) {
+                DRM_DEBUG("framebuffer not tiled, disabling compression\n");
+                goto out_disable;
+        }
+
+        if (dev_priv->display.fbc_enabled(crtc)) {
+                /* We can re-enable it in this case, but need to update pitch */
+                if (fb->pitch > dev_priv->cfb_pitch)
+                        dev_priv->display.disable_fbc(dev);
+                if (obj_priv->fence_reg != dev_priv->cfb_fence)
+                        dev_priv->display.disable_fbc(dev);
+                if (plane != dev_priv->cfb_plane)
+                        dev_priv->display.disable_fbc(dev);
+        }
+
+        if (!dev_priv->display.fbc_enabled(crtc)) {
+                /* Now try to turn it back on if possible */
+                dev_priv->display.enable_fbc(crtc, 500);
+        }
+
+        return;
+
+out_disable:
+        DRM_DEBUG("unsupported config, disabling FBC\n");
+        /* Multiple disables should be harmless */
+        if (dev_priv->display.fbc_enabled(crtc))
+                dev_priv->display.disable_fbc(dev);
+}
+
 static int
 intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                     struct drm_framebuffer *old_fb)
@@ -964,12 +1201,13 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_i915_gem_object *obj_priv;
         struct drm_gem_object *obj;
         int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
         unsigned long Start, Offset;
-        int dspbase = (pipe == 0 ? DSPAADDR : DSPBADDR);
-        int dspsurf = (pipe == 0 ? DSPASURF : DSPBSURF);
-        int dspstride = (pipe == 0) ? DSPASTRIDE : DSPBSTRIDE;
-        int dsptileoff = (pipe == 0 ? DSPATILEOFF : DSPBTILEOFF);
-        int dspcntr_reg = (pipe == 0) ? DSPACNTR : DSPBCNTR;
+        int dspbase = (plane == 0 ? DSPAADDR : DSPBADDR);
+        int dspsurf = (plane == 0 ? DSPASURF : DSPBSURF);
+        int dspstride = (plane == 0) ? DSPASTRIDE : DSPBSTRIDE;
+        int dsptileoff = (plane == 0 ? DSPATILEOFF : DSPBTILEOFF);
+        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
         u32 dspcntr, alignment;
         int ret;
 
@@ -979,12 +1217,12 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 return 0;
         }
 
-        switch (pipe) {
+        switch (plane) {
         case 0:
         case 1:
                 break;
         default:
-                DRM_ERROR("Can't update pipe %d in SAREA\n", pipe);
+                DRM_ERROR("Can't update plane %d in SAREA\n", plane);
                 return -EINVAL;
         }
 
@@ -1086,6 +1324,9 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 I915_READ(dspbase);
         }
 
+        if ((IS_I965G(dev) || plane == 0))
+                intel_update_fbc(crtc, &crtc->mode);
+
         intel_wait_for_vblank(dev);
 
         if (old_fb) {
@@ -1217,6 +1458,7 @@ static void igdng_crtc_dpms(struct drm_crtc *crtc, int mode)
         int transconf_reg = (pipe == 0) ? TRANSACONF : TRANSBCONF;
         int pf_ctl_reg = (pipe == 0) ? PFA_CTL_1 : PFB_CTL_1;
         int pf_win_size = (pipe == 0) ? PFA_WIN_SZ : PFB_WIN_SZ;
+        int pf_win_pos = (pipe == 0) ? PFA_WIN_POS : PFB_WIN_POS;
         int cpu_htot_reg = (pipe == 0) ? HTOTAL_A : HTOTAL_B;
         int cpu_hblank_reg = (pipe == 0) ? HBLANK_A : HBLANK_B;
         int cpu_hsync_reg = (pipe == 0) ? HSYNC_A : HSYNC_B;
@@ -1268,6 +1510,19 @@ static void igdng_crtc_dpms(struct drm_crtc *crtc, int mode)
                         }
                 }
 
+                /* Enable panel fitting for LVDS */
+                if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
+                        temp = I915_READ(pf_ctl_reg);
+                        I915_WRITE(pf_ctl_reg, temp | PF_ENABLE);
+
+                        /* currently full aspect */
+                        I915_WRITE(pf_win_pos, 0);
+
+                        I915_WRITE(pf_win_size,
+                                   (dev_priv->panel_fixed_mode->hdisplay << 16) |
+                                   (dev_priv->panel_fixed_mode->vdisplay));
+                }
+
                 /* Enable CPU pipe */
                 temp = I915_READ(pipeconf_reg);
                 if ((temp & PIPEACONF_ENABLE) == 0) {
@@ -1532,9 +1787,10 @@ static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
         int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
-        int dspcntr_reg = (pipe == 0) ? DSPACNTR : DSPBCNTR;
-        int dspbase_reg = (pipe == 0) ? DSPAADDR : DSPBADDR;
+        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
+        int dspbase_reg = (plane == 0) ? DSPAADDR : DSPBADDR;
         int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
         u32 temp;
 
@@ -1577,6 +1833,9 @@ static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
 
                 intel_crtc_load_lut(crtc);
 
+                if ((IS_I965G(dev) || plane == 0))
+                        intel_update_fbc(crtc, &crtc->mode);
+
                 /* Give the overlay scaler a chance to enable if it's on this pipe */
                 //intel_crtc_dpms_video(crtc, true); TODO
                 intel_update_watermarks(dev);
@@ -1586,6 +1845,10 @@ static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
                 /* Give the overlay scaler a chance to disable if it's on this pipe */
                 //intel_crtc_dpms_video(crtc, FALSE); TODO
 
+                if (dev_priv->cfb_plane == plane &&
+                    dev_priv->display.disable_fbc)
+                        dev_priv->display.disable_fbc(dev);
+
                 /* Disable the VGA plane that we never use */
                 i915_disable_vga(dev);
 
@@ -1634,15 +1897,13 @@ static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
 static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
         struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         bool enabled;
 
-        if (IS_IGDNG(dev))
-                igdng_crtc_dpms(crtc, mode);
-        else
-                i9xx_crtc_dpms(crtc, mode);
+        dev_priv->display.dpms(crtc, mode);
 
         intel_crtc->dpms_mode = mode;
 
@@ -1709,56 +1970,68 @@ static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
         return true;
 }
 
+static int i945_get_display_clock_speed(struct drm_device *dev)
+{
+        return 400000;
+}
 
-/** Returns the core display clock speed for i830 - i945 */
-static int intel_get_core_clock_speed(struct drm_device *dev)
+static int i915_get_display_clock_speed(struct drm_device *dev)
 {
+        return 333000;
+}
 
-        /* Core clock values taken from the published datasheets.
-         * The 830 may go up to 166 Mhz, which we should check.
-         */
-        if (IS_I945G(dev))
-                return 400000;
-        else if (IS_I915G(dev))
-                return 333000;
-        else if (IS_I945GM(dev) || IS_845G(dev) || IS_IGDGM(dev))
-                return 200000;
-        else if (IS_I915GM(dev)) {
-                u16 gcfgc = 0;
+static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
+{
+        return 200000;
+}
 
-                pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
+static int i915gm_get_display_clock_speed(struct drm_device *dev)
+{
+        u16 gcfgc = 0;
 
-                if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
-                        return 133000;
-                else {
-                        switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
-                        case GC_DISPLAY_CLOCK_333_MHZ:
-                                return 333000;
-                        default:
-                        case GC_DISPLAY_CLOCK_190_200_MHZ:
-                                return 190000;
-                        }
-                }
-        } else if (IS_I865G(dev))
-                return 266000;
-        else if (IS_I855(dev)) {
-                u16 hpllcc = 0;
-                /* Assume that the hardware is in the high speed state.  This
-                 * should be the default.
-                 */
-                switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
-                case GC_CLOCK_133_200:
-                case GC_CLOCK_100_200:
-                        return 200000;
-                case GC_CLOCK_166_250:
-                        return 250000;
-                case GC_CLOCK_100_133:
-                        return 133000;
+        pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
+
+        if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
+                return 133000;
+        else {
+                switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
+                case GC_DISPLAY_CLOCK_333_MHZ:
+                        return 333000;
+                default:
+                case GC_DISPLAY_CLOCK_190_200_MHZ:
+                        return 190000;
                 }
-        } else /* 852, 830 */
+        }
+}
+
+static int i865_get_display_clock_speed(struct drm_device *dev)
+{
+        return 266000;
+}
+
+static int i855_get_display_clock_speed(struct drm_device *dev)
+{
+        u16 hpllcc = 0;
+        /* Assume that the hardware is in the high speed state.  This
+         * should be the default.
+         */
+        switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
+        case GC_CLOCK_133_200:
+        case GC_CLOCK_100_200:
+                return 200000;
+        case GC_CLOCK_166_250:
+                return 250000;
+        case GC_CLOCK_100_133:
                 return 133000;
+        }
+
+        /* Shouldn't happen */
+        return 0;
+}
 
-        return 0; /* Silence gcc warning */
+static int i830_get_display_clock_speed(struct drm_device *dev)
+{
+        return 133000;
 }
 
 /**
@@ -1921,7 +2194,14 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 {
         long entries_required, wm_size;
 
-        entries_required = (clock_in_khz * pixel_size * latency_ns) / 1000000;
+        /*
+         * Note: we need to make sure we don't overflow for various clock &
+         * latency values.
+         * clocks go from a few thousand to several hundred thousand.
+         * latency is usually a few thousand
+         */
+        entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
+                1000;
         entries_required /= wm->cacheline_size;
 
         DRM_DEBUG("FIFO entries required for mode: %d\n", entries_required);
@@ -1986,14 +2266,13 @@ static struct cxsr_latency *intel_get_cxsr_latency(int is_desktop, int fsb,
         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
                 latency = &cxsr_latency_table[i];
                 if (is_desktop == latency->is_desktop &&
-                        fsb == latency->fsb_freq && mem == latency->mem_freq)
-                        break;
+                    fsb == latency->fsb_freq && mem == latency->mem_freq)
+                        return latency;
         }
-        if (i >= ARRAY_SIZE(cxsr_latency_table)) {
-                DRM_DEBUG("Unknown FSB/MEM found, disable CxSR\n");
-                return NULL;
-        }
-        return latency;
+
+        DRM_DEBUG("Unknown FSB/MEM found, disable CxSR\n");
+
+        return NULL;
 }
 
 static void igd_disable_cxsr(struct drm_device *dev)
@@ -2084,32 +2363,36 @@ static void igd_enable_cxsr(struct drm_device *dev, unsigned long clock,
  */
 const static int latency_ns = 5000;
 
-static int intel_get_fifo_size(struct drm_device *dev, int plane)
+static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t dsparb = I915_READ(DSPARB);
         int size;
 
-        if (IS_I9XX(dev)) {
-                if (plane == 0)
-                        size = dsparb & 0x7f;
-                else
-                        size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) -
-                                (dsparb & 0x7f);
-        } else if (IS_I85X(dev)) {
-                if (plane == 0)
-                        size = dsparb & 0x1ff;
-                else
-                        size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) -
-                                (dsparb & 0x1ff);
-                size >>= 1; /* Convert to cachelines */
-        } else if (IS_845G(dev)) {
+        if (plane == 0)
                 size = dsparb & 0x7f;
-                size >>= 2; /* Convert to cachelines */
-        } else {
-                size = dsparb & 0x7f;
-                size >>= 1; /* Convert to cachelines */
-        }
+        else
+                size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) -
+                        (dsparb & 0x7f);
+
+        DRM_DEBUG("FIFO size - (0x%08x) %s: %d\n", dsparb, plane ? "B" : "A",
+                  size);
+
+        return size;
+}
+
+static int i85x_get_fifo_size(struct drm_device *dev, int plane)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t dsparb = I915_READ(DSPARB);
+        int size;
+
+        if (plane == 0)
+                size = dsparb & 0x1ff;
+        else
+                size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) -
+                        (dsparb & 0x1ff);
+        size >>= 1; /* Convert to cachelines */
 
         DRM_DEBUG("FIFO size - (0x%08x) %s: %d\n", dsparb, plane ? "B" : "A",
                   size);
@@ -2117,7 +2400,38 @@ static int intel_get_fifo_size(struct drm_device *dev, int plane)
         return size;
 }
 
-static void g4x_update_wm(struct drm_device *dev)
+static int i845_get_fifo_size(struct drm_device *dev, int plane)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t dsparb = I915_READ(DSPARB);
+        int size;
+
+        size = dsparb & 0x7f;
+        size >>= 2; /* Convert to cachelines */
+
+        DRM_DEBUG("FIFO size - (0x%08x) %s: %d\n", dsparb, plane ? "B" : "A",
+                  size);
+
+        return size;
+}
+
+static int i830_get_fifo_size(struct drm_device *dev, int plane)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t dsparb = I915_READ(DSPARB);
+        int size;
+
+        size = dsparb & 0x7f;
+        size >>= 1; /* Convert to cachelines */
+
+        DRM_DEBUG("FIFO size - (0x%08x) %s: %d\n", dsparb, plane ? "B" : "A",
+                  size);
+
+        return size;
+}
+
+static void g4x_update_wm(struct drm_device *dev, int unused, int unused2,
+                          int unused3, int unused4)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 fw_blc_self = I915_READ(FW_BLC_SELF);
@@ -2129,7 +2443,8 @@ static void g4x_update_wm(struct drm_device *dev)
         I915_WRITE(FW_BLC_SELF, fw_blc_self);
 }
 
-static void i965_update_wm(struct drm_device *dev)
+static void i965_update_wm(struct drm_device *dev, int unused, int unused2,
+                           int unused3, int unused4)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -2165,8 +2480,8 @@ static void i9xx_update_wm(struct drm_device *dev, int planea_clock,
         cacheline_size = planea_params.cacheline_size;
 
         /* Update per-plane FIFO sizes */
-        planea_params.fifo_size = intel_get_fifo_size(dev, 0);
-        planeb_params.fifo_size = intel_get_fifo_size(dev, 1);
+        planea_params.fifo_size = dev_priv->display.get_fifo_size(dev, 0);
+        planeb_params.fifo_size = dev_priv->display.get_fifo_size(dev, 1);
 
         planea_wm = intel_calculate_wm(planea_clock, &planea_params,
                                        pixel_size, latency_ns);
@@ -2213,14 +2528,14 @@ static void i9xx_update_wm(struct drm_device *dev, int planea_clock,
         I915_WRITE(FW_BLC2, fwater_hi);
 }
 
-static void i830_update_wm(struct drm_device *dev, int planea_clock,
-                           int pixel_size)
+static void i830_update_wm(struct drm_device *dev, int planea_clock, int unused,
+                           int unused2, int pixel_size)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t fwater_lo = I915_READ(FW_BLC) & ~0xfff;
         int planea_wm;
 
-        i830_wm_info.fifo_size = intel_get_fifo_size(dev, 0);
+        i830_wm_info.fifo_size = dev_priv->display.get_fifo_size(dev, 0);
 
         planea_wm = intel_calculate_wm(planea_clock, &i830_wm_info,
                                        pixel_size, latency_ns);
@@ -2264,6 +2579,7 @@ static void i830_update_wm(struct drm_device *dev, int planea_clock,
   */
 static void intel_update_watermarks(struct drm_device *dev)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc;
         struct intel_crtc *intel_crtc;
         int sr_hdisplay = 0;
@@ -2302,15 +2618,8 @@ static void intel_update_watermarks(struct drm_device *dev)
         else if (IS_IGD(dev))
                 igd_disable_cxsr(dev);
 
-        if (IS_G4X(dev))
-                g4x_update_wm(dev);
-        else if (IS_I965G(dev))
-                i965_update_wm(dev);
-        else if (IS_I9XX(dev) || IS_MOBILE(dev))
-                i9xx_update_wm(dev, planea_clock, planeb_clock, sr_hdisplay,
-                               pixel_size);
-        else
-                i830_update_wm(dev, planea_clock, pixel_size);
+        dev_priv->display.update_wm(dev, planea_clock, planeb_clock,
+                                    sr_hdisplay, pixel_size);
 }
 
 static int intel_crtc_mode_set(struct drm_crtc *crtc,
@@ -2323,10 +2632,11 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
         int fp_reg = (pipe == 0) ? FPA0 : FPB0;
         int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B;
         int dpll_md_reg = (intel_crtc->pipe == 0) ? DPLL_A_MD : DPLL_B_MD;
-        int dspcntr_reg = (pipe == 0) ? DSPACNTR : DSPBCNTR;
+        int dspcntr_reg = (plane == 0) ? DSPACNTR : DSPBCNTR;
         int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
         int htot_reg = (pipe == 0) ? HTOTAL_A : HTOTAL_B;
         int hblank_reg = (pipe == 0) ? HBLANK_A : HBLANK_B;
@@ -2334,8 +2644,8 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
         int vtot_reg = (pipe == 0) ? VTOTAL_A : VTOTAL_B;
         int vblank_reg = (pipe == 0) ? VBLANK_A : VBLANK_B;
         int vsync_reg = (pipe == 0) ? VSYNC_A : VSYNC_B;
-        int dspsize_reg = (pipe == 0) ? DSPASIZE : DSPBSIZE;
-        int dsppos_reg = (pipe == 0) ? DSPAPOS : DSPBPOS;
+        int dspsize_reg = (plane == 0) ? DSPASIZE : DSPBSIZE;
+        int dsppos_reg = (plane == 0) ? DSPAPOS : DSPBPOS;
         int pipesrc_reg = (pipe == 0) ? PIPEASRC : PIPEBSRC;
         int refclk, num_outputs = 0;
         intel_clock_t clock, reduced_clock;
@@ -2568,7 +2878,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
            enable color space conversion */
         if (!IS_IGDNG(dev)) {
                 if (pipe == 0)
-                        dspcntr |= DISPPLANE_SEL_PIPE_A;
+                        dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
                 else
                         dspcntr |= DISPPLANE_SEL_PIPE_B;
         }
@@ -2580,7 +2890,8 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                  * XXX: No double-wide on 915GM pipe B. Is that the only reason for the
                  * pipe == 0 check?
                  */
-                if (mode->clock > intel_get_core_clock_speed(dev) * 9 / 10)
+                if (mode->clock >
+                    dev_priv->display.get_display_clock_speed(dev) * 9 / 10)
                         pipeconf |= PIPEACONF_DOUBLE_WIDE;
                 else
                         pipeconf &= ~PIPEACONF_DOUBLE_WIDE;
@@ -2652,9 +2963,12 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
                 udelay(150);
 
                 if (IS_I965G(dev) && !IS_IGDNG(dev)) {
-                        sdvo_pixel_multiply = adjusted_mode->clock / mode->clock;
-                        I915_WRITE(dpll_md_reg, (0 << DPLL_MD_UDI_DIVIDER_SHIFT) |
+                        if (is_sdvo) {
+                                sdvo_pixel_multiply = adjusted_mode->clock / mode->clock;
+                                I915_WRITE(dpll_md_reg, (0 << DPLL_MD_UDI_DIVIDER_SHIFT) |
                                         ((sdvo_pixel_multiply - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT));
+                        } else
+                                I915_WRITE(dpll_md_reg, 0);
                 } else {
                         /* write it again -- the BIOS does, after all */
                         I915_WRITE(dpll_reg, dpll);
@@ -2734,6 +3048,9 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
         /* Flush the plane changes */
         ret = intel_pipe_set_base(crtc, x, y, old_fb);
 
+        if ((IS_I965G(dev) || plane == 0))
+                intel_update_fbc(crtc, &crtc->mode);
+
         intel_update_watermarks(dev);
 
         drm_vblank_post_modeset(dev, pipe);
@@ -2778,6 +3095,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         struct drm_gem_object *bo;
         struct drm_i915_gem_object *obj_priv;
         int pipe = intel_crtc->pipe;
+        int plane = intel_crtc->plane;
         uint32_t control = (pipe == 0) ? CURACNTR : CURBCNTR;
         uint32_t base = (pipe == 0) ? CURABASE : CURBBASE;
         uint32_t temp = I915_READ(control);
@@ -2863,6 +3181,10 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                         i915_gem_object_unpin(intel_crtc->cursor_bo);
                 drm_gem_object_unreference(intel_crtc->cursor_bo);
         }
+
+        if ((IS_I965G(dev) || plane == 0))
+                intel_update_fbc(crtc, &crtc->mode);
+
         mutex_unlock(&dev->struct_mutex);
 
         intel_crtc->cursor_addr = addr;
@@ -3544,6 +3866,14 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
                 intel_crtc->lut_b[i] = i;
         }
 
+        /* Swap pipes & planes for FBC on pre-965 */
+        intel_crtc->pipe = pipe;
+        intel_crtc->plane = pipe;
+        if (IS_MOBILE(dev) && (IS_I9XX(dev) && !IS_I965G(dev))) {
+                DRM_DEBUG("swapping pipes & planes for FBC\n");
+                intel_crtc->plane = ((pipe == 0) ? 1 : 0);
+        }
+
         intel_crtc->cursor_addr = 0;
         intel_crtc->dpms_mode = DRM_MODE_DPMS_OFF;
         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
@@ -3826,6 +4156,73 @@ void intel_init_clock_gating(struct drm_device *dev)
         }
 }
 
+/* Set up chip specific display functions */
+static void intel_init_display(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /* We always want a DPMS function */
+        if (IS_IGDNG(dev))
+                dev_priv->display.dpms = igdng_crtc_dpms;
+        else
+                dev_priv->display.dpms = i9xx_crtc_dpms;
+
+        /* Only mobile has FBC, leave pointers NULL for other chips */
+        if (IS_MOBILE(dev)) {
+                if (IS_GM45(dev)) {
+                        dev_priv->display.fbc_enabled = g4x_fbc_enabled;
+                        dev_priv->display.enable_fbc = g4x_enable_fbc;
+                        dev_priv->display.disable_fbc = g4x_disable_fbc;
+                } else if (IS_I965GM(dev) || IS_I945GM(dev) || IS_I915GM(dev)) {
+                        dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
+                        dev_priv->display.enable_fbc = i8xx_enable_fbc;
+                        dev_priv->display.disable_fbc = i8xx_disable_fbc;
+                }
+                /* 855GM needs testing */
+        }
+
+        /* Returns the core display clock speed */
+        if (IS_I945G(dev))
+                dev_priv->display.get_display_clock_speed =
+                        i945_get_display_clock_speed;
+        else if (IS_I915G(dev))
+                dev_priv->display.get_display_clock_speed =
+                        i915_get_display_clock_speed;
+        else if (IS_I945GM(dev) || IS_845G(dev) || IS_IGDGM(dev))
+                dev_priv->display.get_display_clock_speed =
+                        i9xx_misc_get_display_clock_speed;
+        else if (IS_I915GM(dev))
+                dev_priv->display.get_display_clock_speed =
+                        i915gm_get_display_clock_speed;
+        else if (IS_I865G(dev))
+                dev_priv->display.get_display_clock_speed =
+                        i865_get_display_clock_speed;
+        else if (IS_I855(dev))
+                dev_priv->display.get_display_clock_speed =
+                        i855_get_display_clock_speed;
+        else /* 852, 830 */
+                dev_priv->display.get_display_clock_speed =
+                        i830_get_display_clock_speed;
+
+        /* For FIFO watermark updates */
+        if (IS_G4X(dev))
+                dev_priv->display.update_wm = g4x_update_wm;
+        else if (IS_I965G(dev))
+                dev_priv->display.update_wm = i965_update_wm;
+        else if (IS_I9XX(dev) || IS_MOBILE(dev)) {
+                dev_priv->display.update_wm = i9xx_update_wm;
+                dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
+        } else {
+                if (IS_I85X(dev))
+                        dev_priv->display.get_fifo_size = i85x_get_fifo_size;
+                else if (IS_845G(dev))
+                        dev_priv->display.get_fifo_size = i845_get_fifo_size;
+                else
+                        dev_priv->display.get_fifo_size = i830_get_fifo_size;
+                dev_priv->display.update_wm = i830_update_wm;
+        }
+}
+
 void intel_modeset_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3839,6 +4236,8 @@ void intel_modeset_init(struct drm_device *dev)
 
         dev->mode_config.funcs = (void *)&intel_mode_funcs;
 
+        intel_init_display(dev);
+
         if (IS_I965G(dev)) {
                 dev->mode_config.max_width = 8192;
                 dev->mode_config.max_height = 8192;
@@ -3904,6 +4303,9 @@ void intel_modeset_cleanup(struct drm_device *dev)
 
         mutex_unlock(&dev->struct_mutex);
 
+        if (dev_priv->display.disable_fbc)
+                dev_priv->display.disable_fbc(dev);
+
         drm_mode_config_cleanup(dev);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 3ebbbabfe59b..8aa4b7f30daa 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -28,6 +28,7 @@
 #include <linux/i2c.h>
 #include <linux/i2c-id.h>
 #include <linux/i2c-algo-bit.h>
+#include "i915_drv.h"
 #include "drm_crtc.h"
 
 #include "drm_crtc_helper.h"
@@ -111,8 +112,8 @@ struct intel_output {
 
 struct intel_crtc {
         struct drm_crtc base;
-        int pipe;
-        int plane;
+        enum pipe pipe;
+        enum plane plane;
         struct drm_gem_object *cursor_bo;
         uint32_t cursor_addr;
         u8 lut_r[256], lut_g[256], lut_b[256];
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index dafc0da1c256..98ae3d73577e 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -27,6 +27,7 @@
  *      Jesse Barnes <jesse.barnes@intel.com>
  */
 
+#include <acpi/button.h>
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include "drmP.h"
@@ -295,6 +296,10 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
                 goto out;
         }
 
+        /* full screen scale for now */
+        if (IS_IGDNG(dev))
+                goto out;
+
         /* 965+ wants fuzzy fitting */
         if (IS_I965G(dev))
                 pfit_control |= (intel_crtc->pipe << PFIT_PIPE_SHIFT) |
@@ -322,8 +327,10 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
          * to register description and PRM.
          * Change the value here to see the borders for debugging
          */
-        I915_WRITE(BCLRPAT_A, 0);
-        I915_WRITE(BCLRPAT_B, 0);
+        if (!IS_IGDNG(dev)) {
+                I915_WRITE(BCLRPAT_A, 0);
+                I915_WRITE(BCLRPAT_B, 0);
+        }
 
         switch (lvds_priv->fitting_mode) {
         case DRM_MODE_SCALE_CENTER:
@@ -572,7 +579,6 @@ static void intel_lvds_mode_set(struct drm_encoder *encoder,
          * settings.
          */
 
-        /* No panel fitting yet, fixme */
         if (IS_IGDNG(dev))
                 return;
 
@@ -585,15 +591,33 @@ static void intel_lvds_mode_set(struct drm_encoder *encoder,
         I915_WRITE(PFIT_CONTROL, lvds_priv->pfit_control);
 }
 
+/* Some lid devices report incorrect lid status, assume they're connected */
+static const struct dmi_system_id bad_lid_status[] = {
+        {
+                .ident = "Aspire One",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+                        DMI_MATCH(DMI_PRODUCT_NAME, "Aspire one"),
+                },
+        },
+        { }
+};
+
 /**
  * Detect the LVDS connection.
  *
- * This always returns CONNECTOR_STATUS_CONNECTED.  This connector should only have
- * been set up if the LVDS was actually connected anyway.
+ * Since LVDS doesn't have hotlug, we use the lid as a proxy.  Open means
+ * connected and closed means disconnected.  We also send hotplug events as
+ * needed, using lid status notification from the input layer.
  */
 static enum drm_connector_status intel_lvds_detect(struct drm_connector *connector)
 {
-        return connector_status_connected;
+        enum drm_connector_status status = connector_status_connected;
+
+        if (!acpi_lid_open() && !dmi_check_system(bad_lid_status))
+                status = connector_status_disconnected;
+
+        return status;
 }
 
 /**
@@ -632,6 +656,24 @@ static int intel_lvds_get_modes(struct drm_connector *connector)
         return 0;
 }
 
+static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
+                            void *unused)
+{
+        struct drm_i915_private *dev_priv =
+                container_of(nb, struct drm_i915_private, lid_notifier);
+        struct drm_device *dev = dev_priv->dev;
+
+        if (acpi_lid_open() && !dev_priv->suspended) {
+                mutex_lock(&dev->mode_config.mutex);
+                drm_helper_resume_force_mode(dev);
+                mutex_unlock(&dev->mode_config.mutex);
+        }
+
+        drm_sysfs_hotplug_event(dev_priv->dev);
+
+        return NOTIFY_OK;
+}
+
 /**
  * intel_lvds_destroy - unregister and free LVDS structures
  * @connector: connector to free
@@ -641,10 +683,14 @@ static int intel_lvds_get_modes(struct drm_connector *connector)
  */
 static void intel_lvds_destroy(struct drm_connector *connector)
 {
+        struct drm_device *dev = connector->dev;
         struct intel_output *intel_output = to_intel_output(connector);
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (intel_output->ddc_bus)
                 intel_i2c_destroy(intel_output->ddc_bus);
+        if (dev_priv->lid_notifier.notifier_call)
+                acpi_lid_notifier_unregister(&dev_priv->lid_notifier);
         drm_sysfs_connector_remove(connector);
         drm_connector_cleanup(connector);
         kfree(connector);
@@ -1011,6 +1057,11 @@ out:
                 pwm |= PWM_PCH_ENABLE;
                 I915_WRITE(BLC_PWM_PCH_CTL1, pwm);
         }
+        dev_priv->lid_notifier.notifier_call = intel_lid_notify;
+        if (acpi_lid_notifier_register(&dev_priv->lid_notifier)) {
+                DRM_DEBUG("lid notifier registration failed\n");
+                dev_priv->lid_notifier.notifier_call = NULL;
+        }
         drm_sysfs_connector_add(connector);
         return;
 
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 0bf28efcf2c1..083bec2e50f9 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -135,6 +135,30 @@ struct intel_sdvo_priv {
         struct intel_sdvo_dtd save_input_dtd_1, save_input_dtd_2;
         struct intel_sdvo_dtd save_output_dtd[16];
         u32 save_SDVOX;
+        /* add the property for the SDVO-TV */
+        struct drm_property *left_property;
+        struct drm_property *right_property;
+        struct drm_property *top_property;
+        struct drm_property *bottom_property;
+        struct drm_property *hpos_property;
+        struct drm_property *vpos_property;
+
+        /* add the property for the SDVO-TV/LVDS */
+        struct drm_property *brightness_property;
+        struct drm_property *contrast_property;
+        struct drm_property *saturation_property;
+        struct drm_property *hue_property;
+
+        /* Add variable to record current setting for the above property */
+        u32     left_margin, right_margin, top_margin, bottom_margin;
+        /* this is to get the range of margin.*/
+        u32     max_hscan,  max_vscan;
+        u32     max_hpos, cur_hpos;
+        u32     max_vpos, cur_vpos;
+        u32     cur_brightness, max_brightness;
+        u32     cur_contrast,   max_contrast;
+        u32     cur_saturation, max_saturation;
+        u32     cur_hue,        max_hue;
 };
 
 static bool
@@ -281,6 +305,31 @@ static const struct _sdvo_cmd_name {
     SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_SDTV_RESOLUTION_SUPPORT),
     SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_SCALED_HDTV_RESOLUTION_SUPPORT),
     SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_SUPPORTED_ENHANCEMENTS),
+    /* Add the op code for SDVO enhancements */
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_POSITION_H),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_POSITION_H),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_POSITION_H),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_POSITION_V),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_POSITION_V),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_POSITION_V),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_SATURATION),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_SATURATION),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_SATURATION),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_HUE),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_HUE),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_HUE),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_CONTRAST),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_CONTRAST),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_CONTRAST),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_BRIGHTNESS),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_BRIGHTNESS),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_BRIGHTNESS),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_OVERSCAN_H),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_OVERSCAN_H),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_OVERSCAN_H),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_MAX_OVERSCAN_V),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_OVERSCAN_V),
+    SDVO_CMD_NAME_ENTRY(SDVO_CMD_SET_OVERSCAN_V),
     /* HDMI op code */
     SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_SUPP_ENCODE),
     SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_ENCODE),
@@ -981,7 +1030,7 @@ static void intel_sdvo_set_tv_format(struct intel_output *output)
 
         status = intel_sdvo_read_response(output, NULL, 0);
         if (status != SDVO_CMD_STATUS_SUCCESS)
-                DRM_DEBUG("%s: Failed to set TV format\n",
+                DRM_DEBUG_KMS("%s: Failed to set TV format\n",
                           SDVO_NAME(sdvo_priv));
 }
 
@@ -1792,6 +1841,45 @@ static int intel_sdvo_get_modes(struct drm_connector *connector)
         return 1;
 }
 
+static
+void intel_sdvo_destroy_enhance_property(struct drm_connector *connector)
+{
+        struct intel_output *intel_output = to_intel_output(connector);
+        struct intel_sdvo_priv *sdvo_priv = intel_output->dev_priv;
+        struct drm_device *dev = connector->dev;
+
+        if (sdvo_priv->is_tv) {
+                if (sdvo_priv->left_property)
+                        drm_property_destroy(dev, sdvo_priv->left_property);
+                if (sdvo_priv->right_property)
+                        drm_property_destroy(dev, sdvo_priv->right_property);
+                if (sdvo_priv->top_property)
+                        drm_property_destroy(dev, sdvo_priv->top_property);
+                if (sdvo_priv->bottom_property)
+                        drm_property_destroy(dev, sdvo_priv->bottom_property);
+                if (sdvo_priv->hpos_property)
+                        drm_property_destroy(dev, sdvo_priv->hpos_property);
+                if (sdvo_priv->vpos_property)
+                        drm_property_destroy(dev, sdvo_priv->vpos_property);
+        }
+        if (sdvo_priv->is_tv) {
+                if (sdvo_priv->saturation_property)
+                        drm_property_destroy(dev,
+                                        sdvo_priv->saturation_property);
+                if (sdvo_priv->contrast_property)
+                        drm_property_destroy(dev,
+                                        sdvo_priv->contrast_property);
+                if (sdvo_priv->hue_property)
+                        drm_property_destroy(dev, sdvo_priv->hue_property);
+        }
+        if (sdvo_priv->is_tv || sdvo_priv->is_lvds) {
+                if (sdvo_priv->brightness_property)
+                        drm_property_destroy(dev,
+                                        sdvo_priv->brightness_property);
+        }
+        return;
+}
+
 static void intel_sdvo_destroy(struct drm_connector *connector)
 {
         struct intel_output *intel_output = to_intel_output(connector);
@@ -1812,6 +1900,9 @@ static void intel_sdvo_destroy(struct drm_connector *connector)
                 drm_property_destroy(connector->dev,
                                      sdvo_priv->tv_format_property);
 
+        if (sdvo_priv->is_tv || sdvo_priv->is_lvds)
+                intel_sdvo_destroy_enhance_property(connector);
+
         drm_sysfs_connector_remove(connector);
         drm_connector_cleanup(connector);
 
@@ -1829,6 +1920,8 @@ intel_sdvo_set_property(struct drm_connector *connector,
         struct drm_crtc *crtc = encoder->crtc;
         int ret = 0;
         bool changed = false;
+        uint8_t cmd, status;
+        uint16_t temp_value;
 
         ret = drm_connector_property_set_value(connector, property, val);
         if (ret < 0)
@@ -1845,11 +1938,102 @@ intel_sdvo_set_property(struct drm_connector *connector,
 
                 sdvo_priv->tv_format_name = sdvo_priv->tv_format_supported[val];
                 changed = true;
-        } else {
-                ret = -EINVAL;
-                goto out;
         }
 
+        if (sdvo_priv->is_tv || sdvo_priv->is_lvds) {
+                cmd = 0;
+                temp_value = val;
+                if (sdvo_priv->left_property == property) {
+                        drm_connector_property_set_value(connector,
+                                sdvo_priv->right_property, val);
+                        if (sdvo_priv->left_margin == temp_value)
+                                goto out;
+
+                        sdvo_priv->left_margin = temp_value;
+                        sdvo_priv->right_margin = temp_value;
+                        temp_value = sdvo_priv->max_hscan -
+                                        sdvo_priv->left_margin;
+                        cmd = SDVO_CMD_SET_OVERSCAN_H;
+                } else if (sdvo_priv->right_property == property) {
+                        drm_connector_property_set_value(connector,
+                                sdvo_priv->left_property, val);
+                        if (sdvo_priv->right_margin == temp_value)
+                                goto out;
+
+                        sdvo_priv->left_margin = temp_value;
+                        sdvo_priv->right_margin = temp_value;
+                        temp_value = sdvo_priv->max_hscan -
+                                sdvo_priv->left_margin;
+                        cmd = SDVO_CMD_SET_OVERSCAN_H;
+                } else if (sdvo_priv->top_property == property) {
+                        drm_connector_property_set_value(connector,
+                                sdvo_priv->bottom_property, val);
+                        if (sdvo_priv->top_margin == temp_value)
+                                goto out;
+
+                        sdvo_priv->top_margin = temp_value;
+                        sdvo_priv->bottom_margin = temp_value;
+                        temp_value = sdvo_priv->max_vscan -
+                                        sdvo_priv->top_margin;
+                        cmd = SDVO_CMD_SET_OVERSCAN_V;
+                } else if (sdvo_priv->bottom_property == property) {
+                        drm_connector_property_set_value(connector,
+                                sdvo_priv->top_property, val);
+                        if (sdvo_priv->bottom_margin == temp_value)
+                                goto out;
+                        sdvo_priv->top_margin = temp_value;
+                        sdvo_priv->bottom_margin = temp_value;
+                        temp_value = sdvo_priv->max_vscan -
+                                        sdvo_priv->top_margin;
+                        cmd = SDVO_CMD_SET_OVERSCAN_V;
+                } else if (sdvo_priv->hpos_property == property) {
+                        if (sdvo_priv->cur_hpos == temp_value)
+                                goto out;
+
+                        cmd = SDVO_CMD_SET_POSITION_H;
+                        sdvo_priv->cur_hpos = temp_value;
+                } else if (sdvo_priv->vpos_property == property) {
+                        if (sdvo_priv->cur_vpos == temp_value)
+                                goto out;
+
+                        cmd = SDVO_CMD_SET_POSITION_V;
+                        sdvo_priv->cur_vpos = temp_value;
+                } else if (sdvo_priv->saturation_property == property) {
+                        if (sdvo_priv->cur_saturation == temp_value)
+                                goto out;
+
+                        cmd = SDVO_CMD_SET_SATURATION;
+                        sdvo_priv->cur_saturation = temp_value;
+                } else if (sdvo_priv->contrast_property == property) {
+                        if (sdvo_priv->cur_contrast == temp_value)
+                                goto out;
+
+                        cmd = SDVO_CMD_SET_CONTRAST;
+                        sdvo_priv->cur_contrast = temp_value;
+                } else if (sdvo_priv->hue_property == property) {
+                        if (sdvo_priv->cur_hue == temp_value)
+                                goto out;
+
+                        cmd = SDVO_CMD_SET_HUE;
+                        sdvo_priv->cur_hue = temp_value;
+                } else if (sdvo_priv->brightness_property == property) {
+                        if (sdvo_priv->cur_brightness == temp_value)
+                                goto out;
+
+                        cmd = SDVO_CMD_SET_BRIGHTNESS;
+                        sdvo_priv->cur_brightness = temp_value;
+                }
+                if (cmd) {
+                        intel_sdvo_write_cmd(intel_output, cmd, &temp_value, 2);
+                        status = intel_sdvo_read_response(intel_output,
+                                                                NULL, 0);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO command \n");
+                                return -EINVAL;
+                        }
+                        changed = true;
+                }
+        }
         if (changed && crtc)
                 drm_crtc_helper_set_mode(crtc, &crtc->mode, crtc->x,
                                 crtc->y, crtc->fb);
@@ -2090,6 +2274,8 @@ intel_sdvo_output_setup(struct intel_output *intel_output, uint16_t flags)
                 sdvo_priv->controlled_output = SDVO_OUTPUT_RGB1;
                 encoder->encoder_type = DRM_MODE_ENCODER_DAC;
                 connector->connector_type = DRM_MODE_CONNECTOR_VGA;
+                intel_output->clone_mask = (1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
+                                        (1 << INTEL_ANALOG_CLONE_BIT);
         } else if (flags & SDVO_OUTPUT_LVDS0) {
 
                 sdvo_priv->controlled_output = SDVO_OUTPUT_LVDS0;
@@ -2176,6 +2362,310 @@ static void intel_sdvo_tv_create_property(struct drm_connector *connector)
 
 }
 
+static void intel_sdvo_create_enhance_property(struct drm_connector *connector)
+{
+        struct intel_output *intel_output = to_intel_output(connector);
+        struct intel_sdvo_priv *sdvo_priv = intel_output->dev_priv;
+        struct intel_sdvo_enhancements_reply sdvo_data;
+        struct drm_device *dev = connector->dev;
+        uint8_t status;
+        uint16_t response, data_value[2];
+
+        intel_sdvo_write_cmd(intel_output, SDVO_CMD_GET_SUPPORTED_ENHANCEMENTS,
+                                                NULL, 0);
+        status = intel_sdvo_read_response(intel_output, &sdvo_data,
+                                        sizeof(sdvo_data));
+        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                DRM_DEBUG_KMS(" incorrect response is returned\n");
+                return;
+        }
+        response = *((uint16_t *)&sdvo_data);
+        if (!response) {
+                DRM_DEBUG_KMS("No enhancement is supported\n");
+                return;
+        }
+        if (sdvo_priv->is_tv) {
+                /* when horizontal overscan is supported, Add the left/right
+                 * property
+                 */
+                if (sdvo_data.overscan_h) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_OVERSCAN_H, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO max "
+                                                "h_overscan\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_OVERSCAN_H, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO h_overscan\n");
+                                return;
+                        }
+                        sdvo_priv->max_hscan = data_value[0];
+                        sdvo_priv->left_margin = data_value[0] - response;
+                        sdvo_priv->right_margin = sdvo_priv->left_margin;
+                        sdvo_priv->left_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "left_margin", 2);
+                        sdvo_priv->left_property->values[0] = 0;
+                        sdvo_priv->left_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->left_property,
+                                                sdvo_priv->left_margin);
+                        sdvo_priv->right_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "right_margin", 2);
+                        sdvo_priv->right_property->values[0] = 0;
+                        sdvo_priv->right_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->right_property,
+                                                sdvo_priv->right_margin);
+                        DRM_DEBUG_KMS("h_overscan: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+                if (sdvo_data.overscan_v) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_OVERSCAN_V, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO max "
+                                                "v_overscan\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_OVERSCAN_V, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO v_overscan\n");
+                                return;
+                        }
+                        sdvo_priv->max_vscan = data_value[0];
+                        sdvo_priv->top_margin = data_value[0] - response;
+                        sdvo_priv->bottom_margin = sdvo_priv->top_margin;
+                        sdvo_priv->top_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "top_margin", 2);
+                        sdvo_priv->top_property->values[0] = 0;
+                        sdvo_priv->top_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->top_property,
+                                                sdvo_priv->top_margin);
+                        sdvo_priv->bottom_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "bottom_margin", 2);
+                        sdvo_priv->bottom_property->values[0] = 0;
+                        sdvo_priv->bottom_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->bottom_property,
+                                                sdvo_priv->bottom_margin);
+                        DRM_DEBUG_KMS("v_overscan: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+                if (sdvo_data.position_h) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_POSITION_H, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO Max h_pos\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_POSITION_H, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO get h_postion\n");
+                                return;
+                        }
+                        sdvo_priv->max_hpos = data_value[0];
+                        sdvo_priv->cur_hpos = response;
+                        sdvo_priv->hpos_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "hpos", 2);
+                        sdvo_priv->hpos_property->values[0] = 0;
+                        sdvo_priv->hpos_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->hpos_property,
+                                                sdvo_priv->cur_hpos);
+                        DRM_DEBUG_KMS("h_position: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+                if (sdvo_data.position_v) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_POSITION_V, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO Max v_pos\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_POSITION_V, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO get v_postion\n");
+                                return;
+                        }
+                        sdvo_priv->max_vpos = data_value[0];
+                        sdvo_priv->cur_vpos = response;
+                        sdvo_priv->vpos_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "vpos", 2);
+                        sdvo_priv->vpos_property->values[0] = 0;
+                        sdvo_priv->vpos_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->vpos_property,
+                                                sdvo_priv->cur_vpos);
+                        DRM_DEBUG_KMS("v_position: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+        }
+        if (sdvo_priv->is_tv) {
+                if (sdvo_data.saturation) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_SATURATION, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO Max sat\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_SATURATION, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO get sat\n");
+                                return;
+                        }
+                        sdvo_priv->max_saturation = data_value[0];
+                        sdvo_priv->cur_saturation = response;
+                        sdvo_priv->saturation_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "saturation", 2);
+                        sdvo_priv->saturation_property->values[0] = 0;
+                        sdvo_priv->saturation_property->values[1] =
+                                                        data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->saturation_property,
+                                                sdvo_priv->cur_saturation);
+                        DRM_DEBUG_KMS("saturation: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+                if (sdvo_data.contrast) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_CONTRAST, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO Max contrast\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_CONTRAST, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO get contrast\n");
+                                return;
+                        }
+                        sdvo_priv->max_contrast = data_value[0];
+                        sdvo_priv->cur_contrast = response;
+                        sdvo_priv->contrast_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "contrast", 2);
+                        sdvo_priv->contrast_property->values[0] = 0;
+                        sdvo_priv->contrast_property->values[1] = data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->contrast_property,
+                                                sdvo_priv->cur_contrast);
+                        DRM_DEBUG_KMS("contrast: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+                if (sdvo_data.hue) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_HUE, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO Max hue\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_HUE, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO get hue\n");
+                                return;
+                        }
+                        sdvo_priv->max_hue = data_value[0];
+                        sdvo_priv->cur_hue = response;
+                        sdvo_priv->hue_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "hue", 2);
+                        sdvo_priv->hue_property->values[0] = 0;
+                        sdvo_priv->hue_property->values[1] =
+                                                        data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->hue_property,
+                                                sdvo_priv->cur_hue);
+                        DRM_DEBUG_KMS("hue: max %d, default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+        }
+        if (sdvo_priv->is_tv || sdvo_priv->is_lvds) {
+                if (sdvo_data.brightness) {
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_MAX_BRIGHTNESS, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &data_value, 4);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO Max bright\n");
+                                return;
+                        }
+                        intel_sdvo_write_cmd(intel_output,
+                                SDVO_CMD_GET_BRIGHTNESS, NULL, 0);
+                        status = intel_sdvo_read_response(intel_output,
+                                &response, 2);
+                        if (status != SDVO_CMD_STATUS_SUCCESS) {
+                                DRM_DEBUG_KMS("Incorrect SDVO get brigh\n");
+                                return;
+                        }
+                        sdvo_priv->max_brightness = data_value[0];
+                        sdvo_priv->cur_brightness = response;
+                        sdvo_priv->brightness_property =
+                                drm_property_create(dev, DRM_MODE_PROP_RANGE,
+                                                "brightness", 2);
+                        sdvo_priv->brightness_property->values[0] = 0;
+                        sdvo_priv->brightness_property->values[1] =
+                                                        data_value[0];
+                        drm_connector_attach_property(connector,
+                                                sdvo_priv->brightness_property,
+                                                sdvo_priv->cur_brightness);
+                        DRM_DEBUG_KMS("brightness: max %d, "
+                                        "default %d, current %d\n",
+                                        data_value[0], data_value[1], response);
+                }
+        }
+        return;
+}
+
 bool intel_sdvo_init(struct drm_device *dev, int output_device)
 {
         struct drm_connector *connector;
@@ -2264,6 +2754,10 @@ bool intel_sdvo_init(struct drm_device *dev, int output_device)
         drm_mode_connector_attach_encoder(&intel_output->base, &intel_output->enc);
         if (sdvo_priv->is_tv)
                 intel_sdvo_tv_create_property(connector);
+
+        if (sdvo_priv->is_tv || sdvo_priv->is_lvds)
+                intel_sdvo_create_enhance_property(connector);
+
         drm_sysfs_connector_add(connector);
 
         intel_sdvo_select_ddc_bus(sdvo_priv);
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 6bedd2fcfc15..737335ff2b21 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -477,8 +477,8 @@ config I2C_PNX
           will be called i2c-pnx.
 
 config I2C_PXA
-        tristate "Intel PXA2XX I2C adapter (EXPERIMENTAL)"
-        depends on EXPERIMENTAL && ARCH_PXA
+        tristate "Intel PXA2XX I2C adapter"
+        depends on ARCH_PXA || ARCH_MMP
         help
           If you have devices in the PXA I2C bus, say yes to this option.
           This driver can also be built as a module.  If so, the module
diff --git a/drivers/idle/i7300_idle.c b/drivers/idle/i7300_idle.c
index 949c97ff57e3..1f20a042a4f5 100644
--- a/drivers/idle/i7300_idle.c
+++ b/drivers/idle/i7300_idle.c
@@ -29,8 +29,8 @@
 
 #include <asm/idle.h>
 
-#include "../dma/ioatdma_hw.h"
-#include "../dma/ioatdma_registers.h"
+#include "../dma/ioat/hw.h"
+#include "../dma/ioat/registers.h"
 
 #define I7300_IDLE_DRIVER_VERSION       "1.55"
 #define I7300_PRINT                     "i7300_idle:"
@@ -126,9 +126,9 @@ static void i7300_idle_ioat_stop(void)
                 udelay(10);
 
                 sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-                        IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+                        IOAT_CHANSTS_STATUS;
 
-                if (sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE)
+                if (sts != IOAT_CHANSTS_ACTIVE)
                         break;
 
         }
@@ -160,9 +160,9 @@ static int __init i7300_idle_ioat_selftest(u8 *ctl,
         udelay(1000);
 
         chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-                        IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+                        IOAT_CHANSTS_STATUS;
 
-        if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE) {
+        if (chan_sts != IOAT_CHANSTS_DONE) {
                 /* Not complete, reset the channel */
                 writeb(IOAT_CHANCMD_RESET,
                        ioat_chanbase + IOAT1_CHANCMD_OFFSET);
@@ -288,9 +288,9 @@ static void __exit i7300_idle_ioat_exit(void)
                        ioat_chanbase + IOAT1_CHANCMD_OFFSET);
 
                 chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-                        IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+                        IOAT_CHANSTS_STATUS;
 
-                if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
+                if (chan_sts != IOAT_CHANSTS_ACTIVE) {
                         writew(0, ioat_chanbase + IOAT_CHANCTRL_OFFSET);
                         break;
                 }
@@ -298,14 +298,14 @@ static void __exit i7300_idle_ioat_exit(void)
         }
 
         chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-                        IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+                        IOAT_CHANSTS_STATUS;
 
         /*
          * We tried to reset multiple times. If IO A/T channel is still active
          * flag an error and return without cleanup. Memory leak is better
          * than random corruption in that extreme error situation.
          */
-        if (chan_sts == IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
+        if (chan_sts == IOAT_CHANSTS_ACTIVE) {
                 printk(KERN_ERR I7300_PRINT "Unable to stop IO A/T channels."
                         " Not freeing resources\n");
                 return;
diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c
index 57a3c6f947b2..4e0f2829e0e5 100644
--- a/drivers/infiniband/core/mad_rmpp.c
+++ b/drivers/infiniband/core/mad_rmpp.c
@@ -37,7 +37,8 @@
 enum rmpp_state {
         RMPP_STATE_ACTIVE,
         RMPP_STATE_TIMEOUT,
-        RMPP_STATE_COMPLETE
+        RMPP_STATE_COMPLETE,
+        RMPP_STATE_CANCELING
 };
 
 struct mad_rmpp_recv {
@@ -87,18 +88,22 @@ void ib_cancel_rmpp_recvs(struct ib_mad_agent_private *agent)
 
         spin_lock_irqsave(&agent->lock, flags);
         list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
+                if (rmpp_recv->state != RMPP_STATE_COMPLETE)
+                        ib_free_recv_mad(rmpp_recv->rmpp_wc);
+                rmpp_recv->state = RMPP_STATE_CANCELING;
+        }
+        spin_unlock_irqrestore(&agent->lock, flags);
+
+        list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
                 cancel_delayed_work(&rmpp_recv->timeout_work);
                 cancel_delayed_work(&rmpp_recv->cleanup_work);
         }
-        spin_unlock_irqrestore(&agent->lock, flags);
 
         flush_workqueue(agent->qp_info->port_priv->wq);
 
         list_for_each_entry_safe(rmpp_recv, temp_rmpp_recv,
                                  &agent->rmpp_list, list) {
                 list_del(&rmpp_recv->list);
-                if (rmpp_recv->state != RMPP_STATE_COMPLETE)
-                        ib_free_recv_mad(rmpp_recv->rmpp_wc);
                 destroy_rmpp_recv(rmpp_recv);
         }
 }
@@ -260,6 +265,10 @@ static void recv_cleanup_handler(struct work_struct *work)
         unsigned long flags;
 
         spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
+        if (rmpp_recv->state == RMPP_STATE_CANCELING) {
+                spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
+                return;
+        }
         list_del(&rmpp_recv->list);
         spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
         destroy_rmpp_recv(rmpp_recv);
diff --git a/drivers/infiniband/hw/mthca/mthca_catas.c b/drivers/infiniband/hw/mthca/mthca_catas.c
index 056b2a4c6970..0aa0110e4b6c 100644
--- a/drivers/infiniband/hw/mthca/mthca_catas.c
+++ b/drivers/infiniband/hw/mthca/mthca_catas.c
@@ -68,11 +68,16 @@ static void catas_reset(struct work_struct *work)
         spin_unlock_irq(&catas_lock);
 
         list_for_each_entry_safe(dev, tmpdev, &tlist, catas_err.list) {
+                struct pci_dev *pdev = dev->pdev;
                 ret = __mthca_restart_one(dev->pdev);
+                /* 'dev' now is not valid */
                 if (ret)
-                        mthca_err(dev, "Reset failed (%d)\n", ret);
-                else
-                        mthca_dbg(dev, "Reset succeeded\n");
+                        printk(KERN_ERR "mthca %s: Reset failed (%d)\n",
+                               pci_name(pdev), ret);
+                else {
+                        struct mthca_dev *d = pci_get_drvdata(pdev);
+                        mthca_dbg(d, "Reset succeeded\n");
+                }
         }
 
         mutex_unlock(&mthca_device_mutex);
diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c
index 538e409d4515..e593af3354b8 100644
--- a/drivers/infiniband/hw/nes/nes_nic.c
+++ b/drivers/infiniband/hw/nes/nes_nic.c
@@ -1566,7 +1566,6 @@ static const struct net_device_ops nes_netdev_ops = {
         .ndo_set_mac_address    = nes_netdev_set_mac_address,
         .ndo_set_multicast_list = nes_netdev_set_multicast_list,
         .ndo_change_mtu         = nes_netdev_change_mtu,
-        .ndo_set_mac_address    = eth_mac_addr,
         .ndo_validate_addr      = eth_validate_addr,
         .ndo_vlan_rx_register   = nes_netdev_vlan_rx_register,
 };
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 25874fc680c9..8763c1ea5eb4 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -362,12 +362,19 @@ void ipoib_mcast_carrier_on_task(struct work_struct *work)
 {
         struct ipoib_dev_priv *priv = container_of(work, struct ipoib_dev_priv,
                                                    carrier_on_task);
+        struct ib_port_attr attr;
 
         /*
          * Take rtnl_lock to avoid racing with ipoib_stop() and
          * turning the carrier back on while a device is being
          * removed.
          */
+        if (ib_query_port(priv->ca, priv->port, &attr) ||
+            attr.state != IB_PORT_ACTIVE) {
+                ipoib_dbg(priv, "Keeping carrier off until IB port is active\n");
+                return;
+        }
+
         rtnl_lock();
         netif_carrier_on(priv->dev);
         rtnl_unlock();
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index 76d6751f89a7..02f4f8f1db6f 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -225,6 +225,7 @@ config INPUT_SGI_BTNS
 config INPUT_WINBOND_CIR
         tristate "Winbond IR remote control"
         depends on X86 && PNP
+        select NEW_LEDS
         select LEDS_CLASS
         select BITREVERSE
         help
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 020f9573fd82..2158377a1359 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -124,6 +124,8 @@ config MD_RAID456
         select MD_RAID6_PQ
         select ASYNC_MEMCPY
         select ASYNC_XOR
+        select ASYNC_PQ
+        select ASYNC_RAID6_RECOV
         ---help---
           A RAID-5 set of N drives with a capacity of C MB per drive provides
           the capacity of C * (N - 1) MB, and protects against a failure
@@ -152,9 +154,33 @@ config MD_RAID456
 
           If unsure, say Y.
 
+config MULTICORE_RAID456
+        bool "RAID-4/RAID-5/RAID-6 Multicore processing (EXPERIMENTAL)"
+        depends on MD_RAID456
+        depends on SMP
+        depends on EXPERIMENTAL
+        ---help---
+          Enable the raid456 module to dispatch per-stripe raid operations to a
+          thread pool.
+
+          If unsure, say N.
+
 config MD_RAID6_PQ
         tristate
 
+config ASYNC_RAID6_TEST
+        tristate "Self test for hardware accelerated raid6 recovery"
+        depends on MD_RAID6_PQ
+        select ASYNC_RAID6_RECOV
+        ---help---
+          This is a one-shot self test that permutes through the
+          recovery of all the possible two disk failure scenarios for a
+          N-disk array.  Recovery is performed with the asynchronous
+          raid6 recovery routines, and will optionally use an offload
+          engine if one is available.
+
+          If unsure, say N.
+
 config MD_MULTIPATH
         tristate "Multipath I/O support"
         depends on BLK_DEV_MD
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 3319c2fec28e..6986b0059d23 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -108,6 +108,8 @@ static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
  * allocated while we're using it
  */
 static int bitmap_checkpage(struct bitmap *bitmap, unsigned long page, int create)
+__releases(bitmap->lock)
+__acquires(bitmap->lock)
 {
         unsigned char *mappage;
 
@@ -325,7 +327,6 @@ static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
         return 0;
 
  bad_alignment:
-        rcu_read_unlock();
         return -EINVAL;
 }
 
@@ -1207,6 +1208,8 @@ void bitmap_daemon_work(struct bitmap *bitmap)
 static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
                                             sector_t offset, int *blocks,
                                             int create)
+__releases(bitmap->lock)
+__acquires(bitmap->lock)
 {
         /* If 'create', we might release the lock and reclaim it.
          * The lock must have been taken with interrupts enabled.
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index ea4842905444..1ceceb334d5e 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -108,6 +108,9 @@ static int linear_congested(void *data, int bits)
         linear_conf_t *conf;
         int i, ret = 0;
 
+        if (mddev_congested(mddev, bits))
+                return 1;
+
         rcu_read_lock();
         conf = rcu_dereference(mddev->private);
 
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 6aa497e4baf8..26ba42a79129 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -262,6 +262,12 @@ static void mddev_resume(mddev_t *mddev)
         mddev->pers->quiesce(mddev, 0);
 }
 
+int mddev_congested(mddev_t *mddev, int bits)
+{
+        return mddev->suspended;
+}
+EXPORT_SYMBOL(mddev_congested);
+
 
 static inline mddev_t *mddev_get(mddev_t *mddev)
 {
@@ -4218,7 +4224,7 @@ static int do_md_run(mddev_t * mddev)
                         set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
                         mddev->sync_thread = md_register_thread(md_do_sync,
                                                                 mddev,
-                                                                "%s_resync");
+                                                                "resync");
                         if (!mddev->sync_thread) {
                                 printk(KERN_ERR "%s: could not start resync"
                                        " thread...\n",
@@ -4575,10 +4581,10 @@ static int get_version(void __user * arg)
 static int get_array_info(mddev_t * mddev, void __user * arg)
 {
         mdu_array_info_t info;
-        int nr,working,active,failed,spare;
+        int nr,working,insync,failed,spare;
         mdk_rdev_t *rdev;
 
-        nr=working=active=failed=spare=0;
+        nr=working=insync=failed=spare=0;
         list_for_each_entry(rdev, &mddev->disks, same_set) {
                 nr++;
                 if (test_bit(Faulty, &rdev->flags))
@@ -4586,7 +4592,7 @@ static int get_array_info(mddev_t * mddev, void __user * arg)
                 else {
                         working++;
                         if (test_bit(In_sync, &rdev->flags))
-                                active++;       
+                                insync++;       
                         else
                                 spare++;
                 }
@@ -4611,7 +4617,7 @@ static int get_array_info(mddev_t * mddev, void __user * arg)
                 info.state = (1<<MD_SB_CLEAN);
         if (mddev->bitmap && mddev->bitmap_offset)
                 info.state = (1<<MD_SB_BITMAP_PRESENT);
-        info.active_disks  = active;
+        info.active_disks  = insync;
         info.working_disks = working;
         info.failed_disks  = failed;
         info.spare_disks   = spare;
@@ -4721,7 +4727,7 @@ static int add_new_disk(mddev_t * mddev, mdu_disk_info_t *info)
                 if (!list_empty(&mddev->disks)) {
                         mdk_rdev_t *rdev0 = list_entry(mddev->disks.next,
                                                         mdk_rdev_t, same_set);
-                        int err = super_types[mddev->major_version]
+                        err = super_types[mddev->major_version]
                                 .load_super(rdev, rdev0, mddev->minor_version);
                         if (err < 0) {
                                 printk(KERN_WARNING 
@@ -5631,7 +5637,10 @@ mdk_thread_t *md_register_thread(void (*run) (mddev_t *), mddev_t *mddev,
         thread->run = run;
         thread->mddev = mddev;
         thread->timeout = MAX_SCHEDULE_TIMEOUT;
-        thread->tsk = kthread_run(md_thread, thread, name, mdname(thread->mddev));
+        thread->tsk = kthread_run(md_thread, thread,
+                                  "%s_%s",
+                                  mdname(thread->mddev),
+                                  name ?: mddev->pers->name);
         if (IS_ERR(thread->tsk)) {
                 kfree(thread);
                 return NULL;
@@ -6745,7 +6754,7 @@ void md_check_recovery(mddev_t *mddev)
                         }
                         mddev->sync_thread = md_register_thread(md_do_sync,
                                                                 mddev,
-                                                                "%s_resync");
+                                                                "resync");
                         if (!mddev->sync_thread) {
                                 printk(KERN_ERR "%s: could not start resync"
                                         " thread...\n", 
diff --git a/drivers/md/md.h b/drivers/md/md.h
index f55d2ff95133..f184b69ef337 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -430,6 +430,7 @@ extern void md_write_end(mddev_t *mddev);
 extern void md_done_sync(mddev_t *mddev, int blocks, int ok);
 extern void md_error(mddev_t *mddev, mdk_rdev_t *rdev);
 
+extern int mddev_congested(mddev_t *mddev, int bits);
 extern void md_super_write(mddev_t *mddev, mdk_rdev_t *rdev,
                            sector_t sector, int size, struct page *page);
 extern void md_super_wait(mddev_t *mddev);
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index d2d3fd54cc68..ee7646f974a0 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -150,7 +150,6 @@ static int multipath_make_request (struct request_queue *q, struct bio * bio)
         }
 
         mp_bh = mempool_alloc(conf->pool, GFP_NOIO);
-        memset(mp_bh, 0, sizeof(*mp_bh));
 
         mp_bh->master_bio = bio;
         mp_bh->mddev = mddev;
@@ -199,6 +198,9 @@ static int multipath_congested(void *data, int bits)
         multipath_conf_t *conf = mddev->private;
         int i, ret = 0;
 
+        if (mddev_congested(mddev, bits))
+                return 1;
+
         rcu_read_lock();
         for (i = 0; i < mddev->raid_disks ; i++) {
                 mdk_rdev_t *rdev = rcu_dereference(conf->multipaths[i].rdev);
@@ -504,7 +506,7 @@ static int multipath_run (mddev_t *mddev)
         }
 
         {
-                mddev->thread = md_register_thread(multipathd, mddev, "%s_multipath");
+                mddev->thread = md_register_thread(multipathd, mddev, NULL);
                 if (!mddev->thread) {
                         printk(KERN_ERR "multipath: couldn't allocate thread"
                                 " for %s\n", mdname(mddev));
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index f845ed98fec9..d3a4ce06015a 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -44,6 +44,9 @@ static int raid0_congested(void *data, int bits)
         mdk_rdev_t **devlist = conf->devlist;
         int i, ret = 0;
 
+        if (mddev_congested(mddev, bits))
+                return 1;
+
         for (i = 0; i < mddev->raid_disks && !ret ; i++) {
                 struct request_queue *q = bdev_get_queue(devlist[i]->bdev);
 
@@ -86,7 +89,7 @@ static void dump_zones(mddev_t *mddev)
 
 static int create_strip_zones(mddev_t *mddev)
 {
-        int i, c, j, err;
+        int i, c, err;
         sector_t curr_zone_end, sectors;
         mdk_rdev_t *smallest, *rdev1, *rdev2, *rdev, **dev;
         struct strip_zone *zone;
@@ -198,6 +201,8 @@ static int create_strip_zones(mddev_t *mddev)
         /* now do the other zones */
         for (i = 1; i < conf->nr_strip_zones; i++)
         {
+                int j;
+
                 zone = conf->strip_zone + i;
                 dev = conf->devlist + i * mddev->raid_disks;
 
@@ -207,7 +212,6 @@ static int create_strip_zones(mddev_t *mddev)
                 c = 0;
 
                 for (j=0; j<cnt; j++) {
-                        char b[BDEVNAME_SIZE];
                         rdev = conf->devlist[j];
                         printk(KERN_INFO "raid0: checking %s ...",
                                 bdevname(rdev->bdev, b));
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index ff7ed3335995..d1b9bd5fd4f6 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -576,6 +576,9 @@ static int raid1_congested(void *data, int bits)
         conf_t *conf = mddev->private;
         int i, ret = 0;
 
+        if (mddev_congested(mddev, bits))
+                return 1;
+
         rcu_read_lock();
         for (i = 0; i < mddev->raid_disks; i++) {
                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
@@ -851,7 +854,7 @@ static int make_request(struct request_queue *q, struct bio * bio)
                 read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset;
                 read_bio->bi_bdev = mirror->rdev->bdev;
                 read_bio->bi_end_io = raid1_end_read_request;
-                read_bio->bi_rw = READ | do_sync;
+                read_bio->bi_rw = READ | (do_sync << BIO_RW_SYNCIO);
                 read_bio->bi_private = r1_bio;
 
                 generic_make_request(read_bio);
@@ -943,7 +946,8 @@ static int make_request(struct request_queue *q, struct bio * bio)
                 mbio->bi_sector = r1_bio->sector + conf->mirrors[i].rdev->data_offset;
                 mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
                 mbio->bi_end_io = raid1_end_write_request;
-                mbio->bi_rw = WRITE | do_barriers | do_sync;
+                mbio->bi_rw = WRITE | (do_barriers << BIO_RW_BARRIER) |
+                        (do_sync << BIO_RW_SYNCIO);
                 mbio->bi_private = r1_bio;
 
                 if (behind_pages) {
@@ -1623,7 +1627,8 @@ static void raid1d(mddev_t *mddev)
                                                 conf->mirrors[i].rdev->data_offset;
                                         bio->bi_bdev = conf->mirrors[i].rdev->bdev;
                                         bio->bi_end_io = raid1_end_write_request;
-                                        bio->bi_rw = WRITE | do_sync;
+                                        bio->bi_rw = WRITE |
+                                                (do_sync << BIO_RW_SYNCIO);
                                         bio->bi_private = r1_bio;
                                         r1_bio->bios[i] = bio;
                                         generic_make_request(bio);
@@ -1672,7 +1677,7 @@ static void raid1d(mddev_t *mddev)
                                 bio->bi_sector = r1_bio->sector + rdev->data_offset;
                                 bio->bi_bdev = rdev->bdev;
                                 bio->bi_end_io = raid1_end_read_request;
-                                bio->bi_rw = READ | do_sync;
+                                bio->bi_rw = READ | (do_sync << BIO_RW_SYNCIO);
                                 bio->bi_private = r1_bio;
                                 unplug = 1;
                                 generic_make_request(bio);
@@ -2047,7 +2052,7 @@ static int run(mddev_t *mddev)
         conf->last_used = j;
 
 
-        mddev->thread = md_register_thread(raid1d, mddev, "%s_raid1");
+        mddev->thread = md_register_thread(raid1d, mddev, NULL);
         if (!mddev->thread) {
                 printk(KERN_ERR
                        "raid1: couldn't allocate thread for %s\n",
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index d0a2152e064f..51c4c5c4d87a 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -631,6 +631,8 @@ static int raid10_congested(void *data, int bits)
         conf_t *conf = mddev->private;
         int i, ret = 0;
 
+        if (mddev_congested(mddev, bits))
+                return 1;
         rcu_read_lock();
         for (i = 0; i < mddev->raid_disks && ret == 0; i++) {
                 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
@@ -882,7 +884,7 @@ static int make_request(struct request_queue *q, struct bio * bio)
                         mirror->rdev->data_offset;
                 read_bio->bi_bdev = mirror->rdev->bdev;
                 read_bio->bi_end_io = raid10_end_read_request;
-                read_bio->bi_rw = READ | do_sync;
+                read_bio->bi_rw = READ | (do_sync << BIO_RW_SYNCIO);
                 read_bio->bi_private = r10_bio;
 
                 generic_make_request(read_bio);
@@ -950,7 +952,7 @@ static int make_request(struct request_queue *q, struct bio * bio)
                         conf->mirrors[d].rdev->data_offset;
                 mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
                 mbio->bi_end_io = raid10_end_write_request;
-                mbio->bi_rw = WRITE | do_sync;
+                mbio->bi_rw = WRITE | (do_sync << BIO_RW_SYNCIO);
                 mbio->bi_private = r10_bio;
 
                 atomic_inc(&r10_bio->remaining);
@@ -1623,7 +1625,7 @@ static void raid10d(mddev_t *mddev)
                                 bio->bi_sector = r10_bio->devs[r10_bio->read_slot].addr
                                         + rdev->data_offset;
                                 bio->bi_bdev = rdev->bdev;
-                                bio->bi_rw = READ | do_sync;
+                                bio->bi_rw = READ | (do_sync << BIO_RW_SYNCIO);
                                 bio->bi_private = r10_bio;
                                 bio->bi_end_io = raid10_end_read_request;
                                 unplug = 1;
@@ -1773,7 +1775,7 @@ static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, i
         max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
         if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
                 /* recovery... the complicated one */
-                int i, j, k;
+                int j, k;
                 r10_bio = NULL;
 
                 for (i=0 ; i<conf->raid_disks; i++)
@@ -2188,7 +2190,7 @@ static int run(mddev_t *mddev)
         }
 
 
-        mddev->thread = md_register_thread(raid10d, mddev, "%s_raid10");
+        mddev->thread = md_register_thread(raid10d, mddev, NULL);
         if (!mddev->thread) {
                 printk(KERN_ERR
                        "raid10: couldn't allocate thread for %s\n",
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 826eb3467357..94829804ab7f 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -47,7 +47,9 @@
 #include <linux/kthread.h>
 #include <linux/raid/pq.h>
 #include <linux/async_tx.h>
+#include <linux/async.h>
 #include <linux/seq_file.h>
+#include <linux/cpu.h>
 #include "md.h"
 #include "raid5.h"
 #include "bitmap.h"
@@ -499,11 +501,18 @@ async_copy_data(int frombio, struct bio *bio, struct page *page,
         struct page *bio_page;
         int i;
         int page_offset;
+        struct async_submit_ctl submit;
+        enum async_tx_flags flags = 0;
 
         if (bio->bi_sector >= sector)
                 page_offset = (signed)(bio->bi_sector - sector) * 512;
         else
                 page_offset = (signed)(sector - bio->bi_sector) * -512;
+
+        if (frombio)
+                flags |= ASYNC_TX_FENCE;
+        init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
+
         bio_for_each_segment(bvl, bio, i) {
                 int len = bio_iovec_idx(bio, i)->bv_len;
                 int clen;
@@ -525,15 +534,14 @@ async_copy_data(int frombio, struct bio *bio, struct page *page,
                         bio_page = bio_iovec_idx(bio, i)->bv_page;
                         if (frombio)
                                 tx = async_memcpy(page, bio_page, page_offset,
-                                        b_offset, clen,
-                                        ASYNC_TX_DEP_ACK,
-                                        tx, NULL, NULL);
+                                                  b_offset, clen, &submit);
                         else
                                 tx = async_memcpy(bio_page, page, b_offset,
-                                        page_offset, clen,
-                                        ASYNC_TX_DEP_ACK,
-                                        tx, NULL, NULL);
+                                                  page_offset, clen, &submit);
                 }
+                /* chain the operations */
+                submit.depend_tx = tx;
+
                 if (clen < len) /* hit end of page */
                         break;
                 page_offset +=  len;
@@ -592,6 +600,7 @@ static void ops_run_biofill(struct stripe_head *sh)
 {
         struct dma_async_tx_descriptor *tx = NULL;
         raid5_conf_t *conf = sh->raid_conf;
+        struct async_submit_ctl submit;
         int i;
 
         pr_debug("%s: stripe %llu\n", __func__,
@@ -615,22 +624,34 @@ static void ops_run_biofill(struct stripe_head *sh)
         }
 
         atomic_inc(&sh->count);
-        async_trigger_callback(ASYNC_TX_DEP_ACK | ASYNC_TX_ACK, tx,
-                ops_complete_biofill, sh);
+        init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
+        async_trigger_callback(&submit);
 }
 
-static void ops_complete_compute5(void *stripe_head_ref)
+static void mark_target_uptodate(struct stripe_head *sh, int target)
 {
-        struct stripe_head *sh = stripe_head_ref;
-        int target = sh->ops.target;
-        struct r5dev *tgt = &sh->dev[target];
+        struct r5dev *tgt;
 
-        pr_debug("%s: stripe %llu\n", __func__,
-                (unsigned long long)sh->sector);
+        if (target < 0)
+                return;
 
+        tgt = &sh->dev[target];
         set_bit(R5_UPTODATE, &tgt->flags);
         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
         clear_bit(R5_Wantcompute, &tgt->flags);
+}
+
+static void ops_complete_compute(void *stripe_head_ref)
+{
+        struct stripe_head *sh = stripe_head_ref;
+
+        pr_debug("%s: stripe %llu\n", __func__,
+                (unsigned long long)sh->sector);
+
+        /* mark the computed target(s) as uptodate */
+        mark_target_uptodate(sh, sh->ops.target);
+        mark_target_uptodate(sh, sh->ops.target2);
+
         clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
         if (sh->check_state == check_state_compute_run)
                 sh->check_state = check_state_compute_result;
@@ -638,16 +659,24 @@ static void ops_complete_compute5(void *stripe_head_ref)
         release_stripe(sh);
 }
 
-static struct dma_async_tx_descriptor *ops_run_compute5(struct stripe_head *sh)
+/* return a pointer to the address conversion region of the scribble buffer */
+static addr_conv_t *to_addr_conv(struct stripe_head *sh,
+                                 struct raid5_percpu *percpu)
+{
+        return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
+}
+
+static struct dma_async_tx_descriptor *
+ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
-        /* kernel stack size limits the total number of disks */
         int disks = sh->disks;
-        struct page *xor_srcs[disks];
+        struct page **xor_srcs = percpu->scribble;
         int target = sh->ops.target;
         struct r5dev *tgt = &sh->dev[target];
         struct page *xor_dest = tgt->page;
         int count = 0;
         struct dma_async_tx_descriptor *tx;
+        struct async_submit_ctl submit;
         int i;
 
         pr_debug("%s: stripe %llu block: %d\n",
@@ -660,17 +689,215 @@ static struct dma_async_tx_descriptor *ops_run_compute5(struct stripe_head *sh)
 
         atomic_inc(&sh->count);
 
+        init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
+                          ops_complete_compute, sh, to_addr_conv(sh, percpu));
         if (unlikely(count == 1))
-                tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE,
-                        0, NULL, ops_complete_compute5, sh);
+                tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
         else
-                tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
-                        ASYNC_TX_XOR_ZERO_DST, NULL,
-                        ops_complete_compute5, sh);
+                tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
 
         return tx;
 }
 
+/* set_syndrome_sources - populate source buffers for gen_syndrome
+ * @srcs - (struct page *) array of size sh->disks
+ * @sh - stripe_head to parse
+ *
+ * Populates srcs in proper layout order for the stripe and returns the
+ * 'count' of sources to be used in a call to async_gen_syndrome.  The P
+ * destination buffer is recorded in srcs[count] and the Q destination
+ * is recorded in srcs[count+1]].
+ */
+static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
+{
+        int disks = sh->disks;
+        int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
+        int d0_idx = raid6_d0(sh);
+        int count;
+        int i;
+
+        for (i = 0; i < disks; i++)
+                srcs[i] = (void *)raid6_empty_zero_page;
+
+        count = 0;
+        i = d0_idx;
+        do {
+                int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
+
+                srcs[slot] = sh->dev[i].page;
+                i = raid6_next_disk(i, disks);
+        } while (i != d0_idx);
+        BUG_ON(count != syndrome_disks);
+
+        return count;
+}
+
+static struct dma_async_tx_descriptor *
+ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
+{
+        int disks = sh->disks;
+        struct page **blocks = percpu->scribble;
+        int target;
+        int qd_idx = sh->qd_idx;
+        struct dma_async_tx_descriptor *tx;
+        struct async_submit_ctl submit;
+        struct r5dev *tgt;
+        struct page *dest;
+        int i;
+        int count;
+
+        if (sh->ops.target < 0)
+                target = sh->ops.target2;
+        else if (sh->ops.target2 < 0)
+                target = sh->ops.target;
+        else
+                /* we should only have one valid target */
+                BUG();
+        BUG_ON(target < 0);
+        pr_debug("%s: stripe %llu block: %d\n",
+                __func__, (unsigned long long)sh->sector, target);
+
+        tgt = &sh->dev[target];
+        BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
+        dest = tgt->page;
+
+        atomic_inc(&sh->count);
+
+        if (target == qd_idx) {
+                count = set_syndrome_sources(blocks, sh);
+                blocks[count] = NULL; /* regenerating p is not necessary */
+                BUG_ON(blocks[count+1] != dest); /* q should already be set */
+                init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
+                                  ops_complete_compute, sh,
+                                  to_addr_conv(sh, percpu));
+                tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
+        } else {
+                /* Compute any data- or p-drive using XOR */
+                count = 0;
+                for (i = disks; i-- ; ) {
+                        if (i == target || i == qd_idx)
+                                continue;
+                        blocks[count++] = sh->dev[i].page;
+                }
+
+                init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
+                                  NULL, ops_complete_compute, sh,
+                                  to_addr_conv(sh, percpu));
+                tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
+        }
+
+        return tx;
+}
+
+static struct dma_async_tx_descriptor *
+ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
+{
+        int i, count, disks = sh->disks;
+        int syndrome_disks = sh->ddf_layout ? disks : disks-2;
+        int d0_idx = raid6_d0(sh);
+        int faila = -1, failb = -1;
+        int target = sh->ops.target;
+        int target2 = sh->ops.target2;
+        struct r5dev *tgt = &sh->dev[target];
+        struct r5dev *tgt2 = &sh->dev[target2];
+        struct dma_async_tx_descriptor *tx;
+        struct page **blocks = percpu->scribble;
+        struct async_submit_ctl submit;
+
+        pr_debug("%s: stripe %llu block1: %d block2: %d\n",
+                 __func__, (unsigned long long)sh->sector, target, target2);
+        BUG_ON(target < 0 || target2 < 0);
+        BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
+        BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
+
+        /* we need to open-code set_syndrome_sources to handle the
+         * slot number conversion for 'faila' and 'failb'
+         */
+        for (i = 0; i < disks ; i++)
+                blocks[i] = (void *)raid6_empty_zero_page;
+        count = 0;
+        i = d0_idx;
+        do {
+                int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
+
+                blocks[slot] = sh->dev[i].page;
+
+                if (i == target)
+                        faila = slot;
+                if (i == target2)
+                        failb = slot;
+                i = raid6_next_disk(i, disks);
+        } while (i != d0_idx);
+        BUG_ON(count != syndrome_disks);
+
+        BUG_ON(faila == failb);
+        if (failb < faila)
+                swap(faila, failb);
+        pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
+                 __func__, (unsigned long long)sh->sector, faila, failb);
+
+        atomic_inc(&sh->count);
+
+        if (failb == syndrome_disks+1) {
+                /* Q disk is one of the missing disks */
+                if (faila == syndrome_disks) {
+                        /* Missing P+Q, just recompute */
+                        init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
+                                          ops_complete_compute, sh,
+                                          to_addr_conv(sh, percpu));
+                        return async_gen_syndrome(blocks, 0, count+2,
+                                                  STRIPE_SIZE, &submit);
+                } else {
+                        struct page *dest;
+                        int data_target;
+                        int qd_idx = sh->qd_idx;
+
+                        /* Missing D+Q: recompute D from P, then recompute Q */
+                        if (target == qd_idx)
+                                data_target = target2;
+                        else
+                                data_target = target;
+
+                        count = 0;
+                        for (i = disks; i-- ; ) {
+                                if (i == data_target || i == qd_idx)
+                                        continue;
+                                blocks[count++] = sh->dev[i].page;
+                        }
+                        dest = sh->dev[data_target].page;
+                        init_async_submit(&submit,
+                                          ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
+                                          NULL, NULL, NULL,
+                                          to_addr_conv(sh, percpu));
+                        tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
+                                       &submit);
+
+                        count = set_syndrome_sources(blocks, sh);
+                        init_async_submit(&submit, ASYNC_TX_FENCE, tx,
+                                          ops_complete_compute, sh,
+                                          to_addr_conv(sh, percpu));
+                        return async_gen_syndrome(blocks, 0, count+2,
+                                                  STRIPE_SIZE, &submit);
+                }
+        } else {
+                init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
+                                  ops_complete_compute, sh,
+                                  to_addr_conv(sh, percpu));
+                if (failb == syndrome_disks) {
+                        /* We're missing D+P. */
+                        return async_raid6_datap_recov(syndrome_disks+2,
+                                                       STRIPE_SIZE, faila,
+                                                       blocks, &submit);
+                } else {
+                        /* We're missing D+D. */
+                        return async_raid6_2data_recov(syndrome_disks+2,
+                                                       STRIPE_SIZE, faila, failb,
+                                                       blocks, &submit);
+                }
+        }
+}
+
+
 static void ops_complete_prexor(void *stripe_head_ref)
 {
         struct stripe_head *sh = stripe_head_ref;
@@ -680,12 +907,13 @@ static void ops_complete_prexor(void *stripe_head_ref)
 }
 
 static struct dma_async_tx_descriptor *
-ops_run_prexor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
+ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
+               struct dma_async_tx_descriptor *tx)
 {
-        /* kernel stack size limits the total number of disks */
         int disks = sh->disks;
-        struct page *xor_srcs[disks];
+        struct page **xor_srcs = percpu->scribble;
         int count = 0, pd_idx = sh->pd_idx, i;
+        struct async_submit_ctl submit;
 
         /* existing parity data subtracted */
         struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
@@ -700,9 +928,9 @@ ops_run_prexor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
                         xor_srcs[count++] = dev->page;
         }
 
-        tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
-                ASYNC_TX_DEP_ACK | ASYNC_TX_XOR_DROP_DST, tx,
-                ops_complete_prexor, sh);
+        init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
+                          ops_complete_prexor, sh, to_addr_conv(sh, percpu));
+        tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
 
         return tx;
 }
@@ -742,17 +970,21 @@ ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
         return tx;
 }
 
-static void ops_complete_postxor(void *stripe_head_ref)
+static void ops_complete_reconstruct(void *stripe_head_ref)
 {
         struct stripe_head *sh = stripe_head_ref;
-        int disks = sh->disks, i, pd_idx = sh->pd_idx;
+        int disks = sh->disks;
+        int pd_idx = sh->pd_idx;
+        int qd_idx = sh->qd_idx;
+        int i;
 
         pr_debug("%s: stripe %llu\n", __func__,
                 (unsigned long long)sh->sector);
 
         for (i = disks; i--; ) {
                 struct r5dev *dev = &sh->dev[i];
-                if (dev->written || i == pd_idx)
+
+                if (dev->written || i == pd_idx || i == qd_idx)
                         set_bit(R5_UPTODATE, &dev->flags);
         }
 
@@ -770,12 +1002,12 @@ static void ops_complete_postxor(void *stripe_head_ref)
 }
 
 static void
-ops_run_postxor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
+ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
+                     struct dma_async_tx_descriptor *tx)
 {
-        /* kernel stack size limits the total number of disks */
         int disks = sh->disks;
-        struct page *xor_srcs[disks];
-
+        struct page **xor_srcs = percpu->scribble;
+        struct async_submit_ctl submit;
         int count = 0, pd_idx = sh->pd_idx, i;
         struct page *xor_dest;
         int prexor = 0;
@@ -809,18 +1041,36 @@ ops_run_postxor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
          * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
          * for the synchronous xor case
          */
-        flags = ASYNC_TX_DEP_ACK | ASYNC_TX_ACK |
+        flags = ASYNC_TX_ACK |
                 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
 
         atomic_inc(&sh->count);
 
-        if (unlikely(count == 1)) {
-                flags &= ~(ASYNC_TX_XOR_DROP_DST | ASYNC_TX_XOR_ZERO_DST);
-                tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE,
-                        flags, tx, ops_complete_postxor, sh);
-        } else
-                tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
-                        flags, tx, ops_complete_postxor, sh);
+        init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
+                          to_addr_conv(sh, percpu));
+        if (unlikely(count == 1))
+                tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
+        else
+                tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
+}
+
+static void
+ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
+                     struct dma_async_tx_descriptor *tx)
+{
+        struct async_submit_ctl submit;
+        struct page **blocks = percpu->scribble;
+        int count;
+
+        pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
+
+        count = set_syndrome_sources(blocks, sh);
+
+        atomic_inc(&sh->count);
+
+        init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
+                          sh, to_addr_conv(sh, percpu));
+        async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
 }
 
 static void ops_complete_check(void *stripe_head_ref)
@@ -835,63 +1085,115 @@ static void ops_complete_check(void *stripe_head_ref)
         release_stripe(sh);
 }
 
-static void ops_run_check(struct stripe_head *sh)
+static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
-        /* kernel stack size limits the total number of disks */
         int disks = sh->disks;
-        struct page *xor_srcs[disks];
+        int pd_idx = sh->pd_idx;
+        int qd_idx = sh->qd_idx;
+        struct page *xor_dest;
+        struct page **xor_srcs = percpu->scribble;
         struct dma_async_tx_descriptor *tx;
-
-        int count = 0, pd_idx = sh->pd_idx, i;
-        struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
+        struct async_submit_ctl submit;
+        int count;
+        int i;
 
         pr_debug("%s: stripe %llu\n", __func__,
                 (unsigned long long)sh->sector);
 
+        count = 0;
+        xor_dest = sh->dev[pd_idx].page;
+        xor_srcs[count++] = xor_dest;
         for (i = disks; i--; ) {
-                struct r5dev *dev = &sh->dev[i];
-                if (i != pd_idx)
-                        xor_srcs[count++] = dev->page;
+                if (i == pd_idx || i == qd_idx)
+                        continue;
+                xor_srcs[count++] = sh->dev[i].page;
         }
 
-        tx = async_xor_zero_sum(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
-                &sh->ops.zero_sum_result, 0, NULL, NULL, NULL);
+        init_async_submit(&submit, 0, NULL, NULL, NULL,
+                          to_addr_conv(sh, percpu));
+        tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
+                           &sh->ops.zero_sum_result, &submit);
+
+        atomic_inc(&sh->count);
+        init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
+        tx = async_trigger_callback(&submit);
+}
+
+static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
+{
+        struct page **srcs = percpu->scribble;
+        struct async_submit_ctl submit;
+        int count;
+
+        pr_debug("%s: stripe %llu checkp: %d\n", __func__,
+                (unsigned long long)sh->sector, checkp);
+
+        count = set_syndrome_sources(srcs, sh);
+        if (!checkp)
+                srcs[count] = NULL;
 
         atomic_inc(&sh->count);
-        tx = async_trigger_callback(ASYNC_TX_DEP_ACK | ASYNC_TX_ACK, tx,
-                ops_complete_check, sh);
+        init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
+                          sh, to_addr_conv(sh, percpu));
+        async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
+                           &sh->ops.zero_sum_result, percpu->spare_page, &submit);
 }
 
-static void raid5_run_ops(struct stripe_head *sh, unsigned long ops_request)
+static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
 {
         int overlap_clear = 0, i, disks = sh->disks;
         struct dma_async_tx_descriptor *tx = NULL;
+        raid5_conf_t *conf = sh->raid_conf;
+        int level = conf->level;
+        struct raid5_percpu *percpu;
+        unsigned long cpu;
 
+        cpu = get_cpu();
+        percpu = per_cpu_ptr(conf->percpu, cpu);
         if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
                 ops_run_biofill(sh);
                 overlap_clear++;
         }
 
         if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
-                tx = ops_run_compute5(sh);
-                /* terminate the chain if postxor is not set to be run */
-                if (tx && !test_bit(STRIPE_OP_POSTXOR, &ops_request))
+                if (level < 6)
+                        tx = ops_run_compute5(sh, percpu);
+                else {
+                        if (sh->ops.target2 < 0 || sh->ops.target < 0)
+                                tx = ops_run_compute6_1(sh, percpu);
+                        else
+                                tx = ops_run_compute6_2(sh, percpu);
+                }
+                /* terminate the chain if reconstruct is not set to be run */
+                if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
                         async_tx_ack(tx);
         }
 
         if (test_bit(STRIPE_OP_PREXOR, &ops_request))
-                tx = ops_run_prexor(sh, tx);
+                tx = ops_run_prexor(sh, percpu, tx);
 
         if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
                 tx = ops_run_biodrain(sh, tx);
                 overlap_clear++;
         }
 
-        if (test_bit(STRIPE_OP_POSTXOR, &ops_request))
-                ops_run_postxor(sh, tx);
+        if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
+                if (level < 6)
+                        ops_run_reconstruct5(sh, percpu, tx);
+                else
+                        ops_run_reconstruct6(sh, percpu, tx);
+        }
 
-        if (test_bit(STRIPE_OP_CHECK, &ops_request))
-                ops_run_check(sh);
+        if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
+                if (sh->check_state == check_state_run)
+                        ops_run_check_p(sh, percpu);
+                else if (sh->check_state == check_state_run_q)
+                        ops_run_check_pq(sh, percpu, 0);
+                else if (sh->check_state == check_state_run_pq)
+                        ops_run_check_pq(sh, percpu, 1);
+                else
+                        BUG();
+        }
 
         if (overlap_clear)
                 for (i = disks; i--; ) {
@@ -899,6 +1201,7 @@ static void raid5_run_ops(struct stripe_head *sh, unsigned long ops_request)
                         if (test_and_clear_bit(R5_Overlap, &dev->flags))
                                 wake_up(&sh->raid_conf->wait_for_overlap);
                 }
+        put_cpu();
 }
 
 static int grow_one_stripe(raid5_conf_t *conf)
@@ -948,6 +1251,28 @@ static int grow_stripes(raid5_conf_t *conf, int num)
         return 0;
 }
 
+/**
+ * scribble_len - return the required size of the scribble region
+ * @num - total number of disks in the array
+ *
+ * The size must be enough to contain:
+ * 1/ a struct page pointer for each device in the array +2
+ * 2/ room to convert each entry in (1) to its corresponding dma
+ *    (dma_map_page()) or page (page_address()) address.
+ *
+ * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
+ * calculate over all devices (not just the data blocks), using zeros in place
+ * of the P and Q blocks.
+ */
+static size_t scribble_len(int num)
+{
+        size_t len;
+
+        len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
+
+        return len;
+}
+
 static int resize_stripes(raid5_conf_t *conf, int newsize)
 {
         /* Make all the stripes able to hold 'newsize' devices.
@@ -976,6 +1301,7 @@ static int resize_stripes(raid5_conf_t *conf, int newsize)
         struct stripe_head *osh, *nsh;
         LIST_HEAD(newstripes);
         struct disk_info *ndisks;
+        unsigned long cpu;
         int err;
         struct kmem_cache *sc;
         int i;
@@ -1041,7 +1367,7 @@ static int resize_stripes(raid5_conf_t *conf, int newsize)
         /* Step 3.
          * At this point, we are holding all the stripes so the array
          * is completely stalled, so now is a good time to resize
-         * conf->disks.
+         * conf->disks and the scribble region
          */
         ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
         if (ndisks) {
@@ -1052,10 +1378,30 @@ static int resize_stripes(raid5_conf_t *conf, int newsize)
         } else
                 err = -ENOMEM;
 
+        get_online_cpus();
+        conf->scribble_len = scribble_len(newsize);
+        for_each_present_cpu(cpu) {
+                struct raid5_percpu *percpu;
+                void *scribble;
+
+                percpu = per_cpu_ptr(conf->percpu, cpu);
+                scribble = kmalloc(conf->scribble_len, GFP_NOIO);
+
+                if (scribble) {
+                        kfree(percpu->scribble);
+                        percpu->scribble = scribble;
+                } else {
+                        err = -ENOMEM;
+                        break;
+                }
+        }
+        put_online_cpus();
+
         /* Step 4, return new stripes to service */
         while(!list_empty(&newstripes)) {
                 nsh = list_entry(newstripes.next, struct stripe_head, lru);
                 list_del_init(&nsh->lru);
+
                 for (i=conf->raid_disks; i < newsize; i++)
                         if (nsh->dev[i].page == NULL) {
                                 struct page *p = alloc_page(GFP_NOIO);
@@ -1594,258 +1940,13 @@ static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
 }
 
 
-
-/*
- * Copy data between a page in the stripe cache, and one or more bion
- * The page could align with the middle of the bio, or there could be
- * several bion, each with several bio_vecs, which cover part of the page
- * Multiple bion are linked together on bi_next.  There may be extras
- * at the end of this list.  We ignore them.
- */
-static void copy_data(int frombio, struct bio *bio,
-                     struct page *page,
-                     sector_t sector)
-{
-        char *pa = page_address(page);
-        struct bio_vec *bvl;
-        int i;
-        int page_offset;
-
-        if (bio->bi_sector >= sector)
-                page_offset = (signed)(bio->bi_sector - sector) * 512;
-        else
-                page_offset = (signed)(sector - bio->bi_sector) * -512;
-        bio_for_each_segment(bvl, bio, i) {
-                int len = bio_iovec_idx(bio,i)->bv_len;
-                int clen;
-                int b_offset = 0;
-
-                if (page_offset < 0) {
-                        b_offset = -page_offset;
-                        page_offset += b_offset;
-                        len -= b_offset;
-                }
-
-                if (len > 0 && page_offset + len > STRIPE_SIZE)
-                        clen = STRIPE_SIZE - page_offset;
-                else clen = len;
-
-                if (clen > 0) {
-                        char *ba = __bio_kmap_atomic(bio, i, KM_USER0);
-                        if (frombio)
-                                memcpy(pa+page_offset, ba+b_offset, clen);
-                        else
-                                memcpy(ba+b_offset, pa+page_offset, clen);
-                        __bio_kunmap_atomic(ba, KM_USER0);
-                }
-                if (clen < len) /* hit end of page */
-                        break;
-                page_offset +=  len;
-        }
-}
-
-#define check_xor()     do {                                              \
-                                if (count == MAX_XOR_BLOCKS) {            \
-                                xor_blocks(count, STRIPE_SIZE, dest, ptr);\
-                                count = 0;                                \
-                           }                                              \
-                        } while(0)
-
-static void compute_parity6(struct stripe_head *sh, int method)
-{
-        raid5_conf_t *conf = sh->raid_conf;
-        int i, pd_idx, qd_idx, d0_idx, disks = sh->disks, count;
-        int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
-        struct bio *chosen;
-        /**** FIX THIS: This could be very bad if disks is close to 256 ****/
-        void *ptrs[syndrome_disks+2];
-
-        pd_idx = sh->pd_idx;
-        qd_idx = sh->qd_idx;
-        d0_idx = raid6_d0(sh);
-
-        pr_debug("compute_parity, stripe %llu, method %d\n",
-                (unsigned long long)sh->sector, method);
-
-        switch(method) {
-        case READ_MODIFY_WRITE:
-                BUG();          /* READ_MODIFY_WRITE N/A for RAID-6 */
-        case RECONSTRUCT_WRITE:
-                for (i= disks; i-- ;)
-                        if ( i != pd_idx && i != qd_idx && sh->dev[i].towrite ) {
-                                chosen = sh->dev[i].towrite;
-                                sh->dev[i].towrite = NULL;
-
-                                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
-                                        wake_up(&conf->wait_for_overlap);
-
-                                BUG_ON(sh->dev[i].written);
-                                sh->dev[i].written = chosen;
-                        }
-                break;
-        case CHECK_PARITY:
-                BUG();          /* Not implemented yet */
-        }
-
-        for (i = disks; i--;)
-                if (sh->dev[i].written) {
-                        sector_t sector = sh->dev[i].sector;
-                        struct bio *wbi = sh->dev[i].written;
-                        while (wbi && wbi->bi_sector < sector + STRIPE_SECTORS) {
-                                copy_data(1, wbi, sh->dev[i].page, sector);
-                                wbi = r5_next_bio(wbi, sector);
-                        }
-
-                        set_bit(R5_LOCKED, &sh->dev[i].flags);
-                        set_bit(R5_UPTODATE, &sh->dev[i].flags);
-                }
-
-        /* Note that unlike RAID-5, the ordering of the disks matters greatly.*/
-
-        for (i = 0; i < disks; i++)
-                ptrs[i] = (void *)raid6_empty_zero_page;
-
-        count = 0;
-        i = d0_idx;
-        do {
-                int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
-
-                ptrs[slot] = page_address(sh->dev[i].page);
-                if (slot < syndrome_disks &&
-                    !test_bit(R5_UPTODATE, &sh->dev[i].flags)) {
-                        printk(KERN_ERR "block %d/%d not uptodate "
-                               "on parity calc\n", i, count);
-                        BUG();
-                }
-
-                i = raid6_next_disk(i, disks);
-        } while (i != d0_idx);
-        BUG_ON(count != syndrome_disks);
-
-        raid6_call.gen_syndrome(syndrome_disks+2, STRIPE_SIZE, ptrs);
-
-        switch(method) {
-        case RECONSTRUCT_WRITE:
-                set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
-                set_bit(R5_UPTODATE, &sh->dev[qd_idx].flags);
-                set_bit(R5_LOCKED,   &sh->dev[pd_idx].flags);
-                set_bit(R5_LOCKED,   &sh->dev[qd_idx].flags);
-                break;
-        case UPDATE_PARITY:
-                set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
-                set_bit(R5_UPTODATE, &sh->dev[qd_idx].flags);
-                break;
-        }
-}
-
-
-/* Compute one missing block */
-static void compute_block_1(struct stripe_head *sh, int dd_idx, int nozero)
-{
-        int i, count, disks = sh->disks;
-        void *ptr[MAX_XOR_BLOCKS], *dest, *p;
-        int qd_idx = sh->qd_idx;
-
-        pr_debug("compute_block_1, stripe %llu, idx %d\n",
-                (unsigned long long)sh->sector, dd_idx);
-
-        if ( dd_idx == qd_idx ) {
-                /* We're actually computing the Q drive */
-                compute_parity6(sh, UPDATE_PARITY);
-        } else {
-                dest = page_address(sh->dev[dd_idx].page);
-                if (!nozero) memset(dest, 0, STRIPE_SIZE);
-                count = 0;
-                for (i = disks ; i--; ) {
-                        if (i == dd_idx || i == qd_idx)
-                                continue;
-                        p = page_address(sh->dev[i].page);
-                        if (test_bit(R5_UPTODATE, &sh->dev[i].flags))
-                                ptr[count++] = p;
-                        else
-                                printk("compute_block() %d, stripe %llu, %d"
-                                       " not present\n", dd_idx,
-                                       (unsigned long long)sh->sector, i);
-
-                        check_xor();
-                }
-                if (count)
-                        xor_blocks(count, STRIPE_SIZE, dest, ptr);
-                if (!nozero) set_bit(R5_UPTODATE, &sh->dev[dd_idx].flags);
-                else clear_bit(R5_UPTODATE, &sh->dev[dd_idx].flags);
-        }
-}
-
-/* Compute two missing blocks */
-static void compute_block_2(struct stripe_head *sh, int dd_idx1, int dd_idx2)
-{
-        int i, count, disks = sh->disks;
-        int syndrome_disks = sh->ddf_layout ? disks : disks-2;
-        int d0_idx = raid6_d0(sh);
-        int faila = -1, failb = -1;
-        /**** FIX THIS: This could be very bad if disks is close to 256 ****/
-        void *ptrs[syndrome_disks+2];
-
-        for (i = 0; i < disks ; i++)
-                ptrs[i] = (void *)raid6_empty_zero_page;
-        count = 0;
-        i = d0_idx;
-        do {
-                int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
-
-                ptrs[slot] = page_address(sh->dev[i].page);
-
-                if (i == dd_idx1)
-                        faila = slot;
-                if (i == dd_idx2)
-                        failb = slot;
-                i = raid6_next_disk(i, disks);
-        } while (i != d0_idx);
-        BUG_ON(count != syndrome_disks);
-
-        BUG_ON(faila == failb);
-        if ( failb < faila ) { int tmp = faila; faila = failb; failb = tmp; }
-
-        pr_debug("compute_block_2, stripe %llu, idx %d,%d (%d,%d)\n",
-                 (unsigned long long)sh->sector, dd_idx1, dd_idx2,
-                 faila, failb);
-
-        if (failb == syndrome_disks+1) {
-                /* Q disk is one of the missing disks */
-                if (faila == syndrome_disks) {
-                        /* Missing P+Q, just recompute */
-                        compute_parity6(sh, UPDATE_PARITY);
-                        return;
-                } else {
-                        /* We're missing D+Q; recompute D from P */
-                        compute_block_1(sh, ((dd_idx1 == sh->qd_idx) ?
-                                             dd_idx2 : dd_idx1),
-                                        0);
-                        compute_parity6(sh, UPDATE_PARITY); /* Is this necessary? */
-                        return;
-                }
-        }
-
-        /* We're missing D+P or D+D; */
-        if (failb == syndrome_disks) {
-                /* We're missing D+P. */
-                raid6_datap_recov(syndrome_disks+2, STRIPE_SIZE, faila, ptrs);
-        } else {
-                /* We're missing D+D. */
-                raid6_2data_recov(syndrome_disks+2, STRIPE_SIZE, faila, failb,
-                                  ptrs);
-        }
-
-        /* Both the above update both missing blocks */
-        set_bit(R5_UPTODATE, &sh->dev[dd_idx1].flags);
-        set_bit(R5_UPTODATE, &sh->dev[dd_idx2].flags);
-}
-
 static void
-schedule_reconstruction5(struct stripe_head *sh, struct stripe_head_state *s,
+schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
                          int rcw, int expand)
 {
         int i, pd_idx = sh->pd_idx, disks = sh->disks;
+        raid5_conf_t *conf = sh->raid_conf;
+        int level = conf->level;
 
         if (rcw) {
                 /* if we are not expanding this is a proper write request, and
@@ -1858,7 +1959,7 @@ schedule_reconstruction5(struct stripe_head *sh, struct stripe_head_state *s,
                 } else
                         sh->reconstruct_state = reconstruct_state_run;
 
-                set_bit(STRIPE_OP_POSTXOR, &s->ops_request);
+                set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
 
                 for (i = disks; i--; ) {
                         struct r5dev *dev = &sh->dev[i];
@@ -1871,17 +1972,18 @@ schedule_reconstruction5(struct stripe_head *sh, struct stripe_head_state *s,
                                 s->locked++;
                         }
                 }
-                if (s->locked + 1 == disks)
+                if (s->locked + conf->max_degraded == disks)
                         if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
-                                atomic_inc(&sh->raid_conf->pending_full_writes);
+                                atomic_inc(&conf->pending_full_writes);
         } else {
+                BUG_ON(level == 6);
                 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
                         test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
 
                 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
                 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
                 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
-                set_bit(STRIPE_OP_POSTXOR, &s->ops_request);
+                set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
 
                 for (i = disks; i--; ) {
                         struct r5dev *dev = &sh->dev[i];
@@ -1899,13 +2001,22 @@ schedule_reconstruction5(struct stripe_head *sh, struct stripe_head_state *s,
                 }
         }
 
-        /* keep the parity disk locked while asynchronous operations
+        /* keep the parity disk(s) locked while asynchronous operations
          * are in flight
          */
         set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
         clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
         s->locked++;
 
+        if (level == 6) {
+                int qd_idx = sh->qd_idx;
+                struct r5dev *dev = &sh->dev[qd_idx];
+
+                set_bit(R5_LOCKED, &dev->flags);
+                clear_bit(R5_UPTODATE, &dev->flags);
+                s->locked++;
+        }
+
         pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
                 __func__, (unsigned long long)sh->sector,
                 s->locked, s->ops_request);
@@ -1986,13 +2097,6 @@ static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, in
 
 static void end_reshape(raid5_conf_t *conf);
 
-static int page_is_zero(struct page *p)
-{
-        char *a = page_address(p);
-        return ((*(u32*)a) == 0 &&
-                memcmp(a, a+4, STRIPE_SIZE-4)==0);
-}
-
 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
                             struct stripe_head *sh)
 {
@@ -2132,9 +2236,10 @@ static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s,
                         set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
                         set_bit(R5_Wantcompute, &dev->flags);
                         sh->ops.target = disk_idx;
+                        sh->ops.target2 = -1;
                         s->req_compute = 1;
                         /* Careful: from this point on 'uptodate' is in the eye
-                         * of raid5_run_ops which services 'compute' operations
+                         * of raid_run_ops which services 'compute' operations
                          * before writes. R5_Wantcompute flags a block that will
                          * be R5_UPTODATE by the time it is needed for a
                          * subsequent operation.
@@ -2173,61 +2278,104 @@ static void handle_stripe_fill5(struct stripe_head *sh,
         set_bit(STRIPE_HANDLE, &sh->state);
 }
 
-static void handle_stripe_fill6(struct stripe_head *sh,
-                        struct stripe_head_state *s, struct r6_state *r6s,
-                        int disks)
+/* fetch_block6 - checks the given member device to see if its data needs
+ * to be read or computed to satisfy a request.
+ *
+ * Returns 1 when no more member devices need to be checked, otherwise returns
+ * 0 to tell the loop in handle_stripe_fill6 to continue
+ */
+static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s,
+                         struct r6_state *r6s, int disk_idx, int disks)
 {
-        int i;
-        for (i = disks; i--; ) {
-                struct r5dev *dev = &sh->dev[i];
-                if (!test_bit(R5_LOCKED, &dev->flags) &&
-                    !test_bit(R5_UPTODATE, &dev->flags) &&
-                    (dev->toread || (dev->towrite &&
-                     !test_bit(R5_OVERWRITE, &dev->flags)) ||
-                     s->syncing || s->expanding ||
-                     (s->failed >= 1 &&
-                      (sh->dev[r6s->failed_num[0]].toread ||
-                       s->to_write)) ||
-                     (s->failed >= 2 &&
-                      (sh->dev[r6s->failed_num[1]].toread ||
-                       s->to_write)))) {
-                        /* we would like to get this block, possibly
-                         * by computing it, but we might not be able to
+        struct r5dev *dev = &sh->dev[disk_idx];
+        struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]],
+                                  &sh->dev[r6s->failed_num[1]] };
+
+        if (!test_bit(R5_LOCKED, &dev->flags) &&
+            !test_bit(R5_UPTODATE, &dev->flags) &&
+            (dev->toread ||
+             (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
+             s->syncing || s->expanding ||
+             (s->failed >= 1 &&
+              (fdev[0]->toread || s->to_write)) ||
+             (s->failed >= 2 &&
+              (fdev[1]->toread || s->to_write)))) {
+                /* we would like to get this block, possibly by computing it,
+                 * otherwise read it if the backing disk is insync
+                 */
+                BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
+                BUG_ON(test_bit(R5_Wantread, &dev->flags));
+                if ((s->uptodate == disks - 1) &&
+                    (s->failed && (disk_idx == r6s->failed_num[0] ||
+                                   disk_idx == r6s->failed_num[1]))) {
+                        /* have disk failed, and we're requested to fetch it;
+                         * do compute it
                          */
-                        if ((s->uptodate == disks - 1) &&
-                            (s->failed && (i == r6s->failed_num[0] ||
-                                           i == r6s->failed_num[1]))) {
-                                pr_debug("Computing stripe %llu block %d\n",
-                                       (unsigned long long)sh->sector, i);
-                                compute_block_1(sh, i, 0);
-                                s->uptodate++;
-                        } else if ( s->uptodate == disks-2 && s->failed >= 2 ) {
-                                /* Computing 2-failure is *very* expensive; only
-                                 * do it if failed >= 2
-                                 */
-                                int other;
-                                for (other = disks; other--; ) {
-                                        if (other == i)
-                                                continue;
-                                        if (!test_bit(R5_UPTODATE,
-                                              &sh->dev[other].flags))
-                                                break;
-                                }
-                                BUG_ON(other < 0);
-                                pr_debug("Computing stripe %llu blocks %d,%d\n",
-                                       (unsigned long long)sh->sector,
-                                       i, other);
-                                compute_block_2(sh, i, other);
-                                s->uptodate += 2;
-                        } else if (test_bit(R5_Insync, &dev->flags)) {
-                                set_bit(R5_LOCKED, &dev->flags);
-                                set_bit(R5_Wantread, &dev->flags);
-                                s->locked++;
-                                pr_debug("Reading block %d (sync=%d)\n",
-                                        i, s->syncing);
+                        pr_debug("Computing stripe %llu block %d\n",
+                               (unsigned long long)sh->sector, disk_idx);
+                        set_bit(STRIPE_COMPUTE_RUN, &sh->state);
+                        set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
+                        set_bit(R5_Wantcompute, &dev->flags);
+                        sh->ops.target = disk_idx;
+                        sh->ops.target2 = -1; /* no 2nd target */
+                        s->req_compute = 1;
+                        s->uptodate++;
+                        return 1;
+                } else if (s->uptodate == disks-2 && s->failed >= 2) {
+                        /* Computing 2-failure is *very* expensive; only
+                         * do it if failed >= 2
+                         */
+                        int other;
+                        for (other = disks; other--; ) {
+                                if (other == disk_idx)
+                                        continue;
+                                if (!test_bit(R5_UPTODATE,
+                                      &sh->dev[other].flags))
+                                        break;
                         }
+                        BUG_ON(other < 0);
+                        pr_debug("Computing stripe %llu blocks %d,%d\n",
+                               (unsigned long long)sh->sector,
+                               disk_idx, other);
+                        set_bit(STRIPE_COMPUTE_RUN, &sh->state);
+                        set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
+                        set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
+                        set_bit(R5_Wantcompute, &sh->dev[other].flags);
+                        sh->ops.target = disk_idx;
+                        sh->ops.target2 = other;
+                        s->uptodate += 2;
+                        s->req_compute = 1;
+                        return 1;
+                } else if (test_bit(R5_Insync, &dev->flags)) {
+                        set_bit(R5_LOCKED, &dev->flags);
+                        set_bit(R5_Wantread, &dev->flags);
+                        s->locked++;
+                        pr_debug("Reading block %d (sync=%d)\n",
+                                disk_idx, s->syncing);
                 }
         }
+
+        return 0;
+}
+
+/**
+ * handle_stripe_fill6 - read or compute data to satisfy pending requests.
+ */
+static void handle_stripe_fill6(struct stripe_head *sh,
+                        struct stripe_head_state *s, struct r6_state *r6s,
+                        int disks)
+{
+        int i;
+
+        /* look for blocks to read/compute, skip this if a compute
+         * is already in flight, or if the stripe contents are in the
+         * midst of changing due to a write
+         */
+        if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
+            !sh->reconstruct_state)
+                for (i = disks; i--; )
+                        if (fetch_block6(sh, s, r6s, i, disks))
+                                break;
         set_bit(STRIPE_HANDLE, &sh->state);
 }
 
@@ -2361,114 +2509,61 @@ static void handle_stripe_dirtying5(raid5_conf_t *conf,
          */
         /* since handle_stripe can be called at any time we need to handle the
          * case where a compute block operation has been submitted and then a
-         * subsequent call wants to start a write request.  raid5_run_ops only
-         * handles the case where compute block and postxor are requested
+         * subsequent call wants to start a write request.  raid_run_ops only
+         * handles the case where compute block and reconstruct are requested
          * simultaneously.  If this is not the case then new writes need to be
          * held off until the compute completes.
          */
         if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
             (s->locked == 0 && (rcw == 0 || rmw == 0) &&
             !test_bit(STRIPE_BIT_DELAY, &sh->state)))
-                schedule_reconstruction5(sh, s, rcw == 0, 0);
+                schedule_reconstruction(sh, s, rcw == 0, 0);
 }
 
 static void handle_stripe_dirtying6(raid5_conf_t *conf,
                 struct stripe_head *sh, struct stripe_head_state *s,
                 struct r6_state *r6s, int disks)
 {
-        int rcw = 0, must_compute = 0, pd_idx = sh->pd_idx, i;
+        int rcw = 0, pd_idx = sh->pd_idx, i;
         int qd_idx = sh->qd_idx;
+
+        set_bit(STRIPE_HANDLE, &sh->state);
         for (i = disks; i--; ) {
                 struct r5dev *dev = &sh->dev[i];
-                /* Would I have to read this buffer for reconstruct_write */
-                if (!test_bit(R5_OVERWRITE, &dev->flags)
-                    && i != pd_idx && i != qd_idx
-                    && (!test_bit(R5_LOCKED, &dev->flags)
-                            ) &&
-                    !test_bit(R5_UPTODATE, &dev->flags)) {
-                        if (test_bit(R5_Insync, &dev->flags)) rcw++;
-                        else {
-                                pr_debug("raid6: must_compute: "
-                                        "disk %d flags=%#lx\n", i, dev->flags);
-                                must_compute++;
+                /* check if we haven't enough data */
+                if (!test_bit(R5_OVERWRITE, &dev->flags) &&
+                    i != pd_idx && i != qd_idx &&
+                    !test_bit(R5_LOCKED, &dev->flags) &&
+                    !(test_bit(R5_UPTODATE, &dev->flags) ||
+                      test_bit(R5_Wantcompute, &dev->flags))) {
+                        rcw++;
+                        if (!test_bit(R5_Insync, &dev->flags))
+                                continue; /* it's a failed drive */
+
+                        if (
+                          test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
+                                pr_debug("Read_old stripe %llu "
+                                        "block %d for Reconstruct\n",
+                                     (unsigned long long)sh->sector, i);
+                                set_bit(R5_LOCKED, &dev->flags);
+                                set_bit(R5_Wantread, &dev->flags);
+                                s->locked++;
+                        } else {
+                                pr_debug("Request delayed stripe %llu "
+                                        "block %d for Reconstruct\n",
+                                     (unsigned long long)sh->sector, i);
+                                set_bit(STRIPE_DELAYED, &sh->state);
+                                set_bit(STRIPE_HANDLE, &sh->state);
                         }
                 }
         }
-        pr_debug("for sector %llu, rcw=%d, must_compute=%d\n",
-               (unsigned long long)sh->sector, rcw, must_compute);
-        set_bit(STRIPE_HANDLE, &sh->state);
-
-        if (rcw > 0)
-                /* want reconstruct write, but need to get some data */
-                for (i = disks; i--; ) {
-                        struct r5dev *dev = &sh->dev[i];
-                        if (!test_bit(R5_OVERWRITE, &dev->flags)
-                            && !(s->failed == 0 && (i == pd_idx || i == qd_idx))
-                            && !test_bit(R5_LOCKED, &dev->flags) &&
-                            !test_bit(R5_UPTODATE, &dev->flags) &&
-                            test_bit(R5_Insync, &dev->flags)) {
-                                if (
-                                  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
-                                        pr_debug("Read_old stripe %llu "
-                                                "block %d for Reconstruct\n",
-                                             (unsigned long long)sh->sector, i);
-                                        set_bit(R5_LOCKED, &dev->flags);
-                                        set_bit(R5_Wantread, &dev->flags);
-                                        s->locked++;
-                                } else {
-                                        pr_debug("Request delayed stripe %llu "
-                                                "block %d for Reconstruct\n",
-                                             (unsigned long long)sh->sector, i);
-                                        set_bit(STRIPE_DELAYED, &sh->state);
-                                        set_bit(STRIPE_HANDLE, &sh->state);
-                                }
-                        }
-                }
         /* now if nothing is locked, and if we have enough data, we can start a
          * write request
          */
-        if (s->locked == 0 && rcw == 0 &&
+        if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
+            s->locked == 0 && rcw == 0 &&
             !test_bit(STRIPE_BIT_DELAY, &sh->state)) {
-                if (must_compute > 0) {
-                        /* We have failed blocks and need to compute them */
-                        switch (s->failed) {
-                        case 0:
-                                BUG();
-                        case 1:
-                                compute_block_1(sh, r6s->failed_num[0], 0);
-                                break;
-                        case 2:
-                                compute_block_2(sh, r6s->failed_num[0],
-                                                r6s->failed_num[1]);
-                                break;
-                        default: /* This request should have been failed? */
-                                BUG();
-                        }
-                }
-
-                pr_debug("Computing parity for stripe %llu\n",
-                        (unsigned long long)sh->sector);
-                compute_parity6(sh, RECONSTRUCT_WRITE);
-                /* now every locked buffer is ready to be written */
-                for (i = disks; i--; )
-                        if (test_bit(R5_LOCKED, &sh->dev[i].flags)) {
-                                pr_debug("Writing stripe %llu block %d\n",
-                                       (unsigned long long)sh->sector, i);
-                                s->locked++;
-                                set_bit(R5_Wantwrite, &sh->dev[i].flags);
-                        }
-                if (s->locked == disks)
-                        if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
-                                atomic_inc(&conf->pending_full_writes);
-                /* after a RECONSTRUCT_WRITE, the stripe MUST be in-sync */
-                set_bit(STRIPE_INSYNC, &sh->state);
-
-                if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
-                        atomic_dec(&conf->preread_active_stripes);
-                        if (atomic_read(&conf->preread_active_stripes) <
-                            IO_THRESHOLD)
-                                md_wakeup_thread(conf->mddev->thread);
-                }
+                schedule_reconstruction(sh, s, 1, 0);
         }
 }
 
@@ -2527,7 +2622,7 @@ static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
                  * we are done.  Otherwise update the mismatch count and repair
                  * parity if !MD_RECOVERY_CHECK
                  */
-                if (sh->ops.zero_sum_result == 0)
+                if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
                         /* parity is correct (on disc,
                          * not in buffer any more)
                          */
@@ -2544,6 +2639,7 @@ static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
                                 set_bit(R5_Wantcompute,
                                         &sh->dev[sh->pd_idx].flags);
                                 sh->ops.target = sh->pd_idx;
+                                sh->ops.target2 = -1;
                                 s->uptodate++;
                         }
                 }
@@ -2560,67 +2656,74 @@ static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
 
 
 static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
-                                struct stripe_head_state *s,
-                                struct r6_state *r6s, struct page *tmp_page,
-                                int disks)
+                                  struct stripe_head_state *s,
+                                  struct r6_state *r6s, int disks)
 {
-        int update_p = 0, update_q = 0;
-        struct r5dev *dev;
         int pd_idx = sh->pd_idx;
         int qd_idx = sh->qd_idx;
+        struct r5dev *dev;
 
         set_bit(STRIPE_HANDLE, &sh->state);
 
         BUG_ON(s->failed > 2);
-        BUG_ON(s->uptodate < disks);
+
         /* Want to check and possibly repair P and Q.
          * However there could be one 'failed' device, in which
          * case we can only check one of them, possibly using the
          * other to generate missing data
          */
 
-        /* If !tmp_page, we cannot do the calculations,
-         * but as we have set STRIPE_HANDLE, we will soon be called
-         * by stripe_handle with a tmp_page - just wait until then.
-         */
-        if (tmp_page) {
+        switch (sh->check_state) {
+        case check_state_idle:
+                /* start a new check operation if there are < 2 failures */
                 if (s->failed == r6s->q_failed) {
-                        /* The only possible failed device holds 'Q', so it
+                        /* The only possible failed device holds Q, so it
                          * makes sense to check P (If anything else were failed,
                          * we would have used P to recreate it).
                          */
-                        compute_block_1(sh, pd_idx, 1);
-                        if (!page_is_zero(sh->dev[pd_idx].page)) {
-                                compute_block_1(sh, pd_idx, 0);
-                                update_p = 1;
-                        }
+                        sh->check_state = check_state_run;
                 }
                 if (!r6s->q_failed && s->failed < 2) {
-                        /* q is not failed, and we didn't use it to generate
+                        /* Q is not failed, and we didn't use it to generate
                          * anything, so it makes sense to check it
                          */
-                        memcpy(page_address(tmp_page),
-                               page_address(sh->dev[qd_idx].page),
-                               STRIPE_SIZE);
-                        compute_parity6(sh, UPDATE_PARITY);
-                        if (memcmp(page_address(tmp_page),
-                                   page_address(sh->dev[qd_idx].page),
-                                   STRIPE_SIZE) != 0) {
-                                clear_bit(STRIPE_INSYNC, &sh->state);
-                                update_q = 1;
-                        }
+                        if (sh->check_state == check_state_run)
+                                sh->check_state = check_state_run_pq;
+                        else
+                                sh->check_state = check_state_run_q;
                 }
-                if (update_p || update_q) {
-                        conf->mddev->resync_mismatches += STRIPE_SECTORS;
-                        if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
-                                /* don't try to repair!! */
-                                update_p = update_q = 0;
+
+                /* discard potentially stale zero_sum_result */
+                sh->ops.zero_sum_result = 0;
+
+                if (sh->check_state == check_state_run) {
+                        /* async_xor_zero_sum destroys the contents of P */
+                        clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
+                        s->uptodate--;
+                }
+                if (sh->check_state >= check_state_run &&
+                    sh->check_state <= check_state_run_pq) {
+                        /* async_syndrome_zero_sum preserves P and Q, so
+                         * no need to mark them !uptodate here
+                         */
+                        set_bit(STRIPE_OP_CHECK, &s->ops_request);
+                        break;
                 }
 
+                /* we have 2-disk failure */
+                BUG_ON(s->failed != 2);
+                /* fall through */
+        case check_state_compute_result:
+                sh->check_state = check_state_idle;
+
+                /* check that a write has not made the stripe insync */
+                if (test_bit(STRIPE_INSYNC, &sh->state))
+                        break;
+
                 /* now write out any block on a failed drive,
-                 * or P or Q if they need it
+                 * or P or Q if they were recomputed
                  */
-
+                BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
                 if (s->failed == 2) {
                         dev = &sh->dev[r6s->failed_num[1]];
                         s->locked++;
@@ -2633,14 +2736,13 @@ static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
                         set_bit(R5_LOCKED, &dev->flags);
                         set_bit(R5_Wantwrite, &dev->flags);
                 }
-
-                if (update_p) {
+                if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
                         dev = &sh->dev[pd_idx];
                         s->locked++;
                         set_bit(R5_LOCKED, &dev->flags);
                         set_bit(R5_Wantwrite, &dev->flags);
                 }
-                if (update_q) {
+                if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
                         dev = &sh->dev[qd_idx];
                         s->locked++;
                         set_bit(R5_LOCKED, &dev->flags);
@@ -2649,6 +2751,70 @@ static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
                 clear_bit(STRIPE_DEGRADED, &sh->state);
 
                 set_bit(STRIPE_INSYNC, &sh->state);
+                break;
+        case check_state_run:
+        case check_state_run_q:
+        case check_state_run_pq:
+                break; /* we will be called again upon completion */
+        case check_state_check_result:
+                sh->check_state = check_state_idle;
+
+                /* handle a successful check operation, if parity is correct
+                 * we are done.  Otherwise update the mismatch count and repair
+                 * parity if !MD_RECOVERY_CHECK
+                 */
+                if (sh->ops.zero_sum_result == 0) {
+                        /* both parities are correct */
+                        if (!s->failed)
+                                set_bit(STRIPE_INSYNC, &sh->state);
+                        else {
+                                /* in contrast to the raid5 case we can validate
+                                 * parity, but still have a failure to write
+                                 * back
+                                 */
+                                sh->check_state = check_state_compute_result;
+                                /* Returning at this point means that we may go
+                                 * off and bring p and/or q uptodate again so
+                                 * we make sure to check zero_sum_result again
+                                 * to verify if p or q need writeback
+                                 */
+                        }
+                } else {
+                        conf->mddev->resync_mismatches += STRIPE_SECTORS;
+                        if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
+                                /* don't try to repair!! */
+                                set_bit(STRIPE_INSYNC, &sh->state);
+                        else {
+                                int *target = &sh->ops.target;
+
+                                sh->ops.target = -1;
+                                sh->ops.target2 = -1;
+                                sh->check_state = check_state_compute_run;
+                                set_bit(STRIPE_COMPUTE_RUN, &sh->state);
+                                set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
+                                if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
+                                        set_bit(R5_Wantcompute,
+                                                &sh->dev[pd_idx].flags);
+                                        *target = pd_idx;
+                                        target = &sh->ops.target2;
+                                        s->uptodate++;
+                                }
+                                if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
+                                        set_bit(R5_Wantcompute,
+                                                &sh->dev[qd_idx].flags);
+                                        *target = qd_idx;
+                                        s->uptodate++;
+                                }
+                        }
+                }
+                break;
+        case check_state_compute_run:
+                break;
+        default:
+                printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
+                       __func__, sh->check_state,
+                       (unsigned long long) sh->sector);
+                BUG();
         }
 }
 
@@ -2666,6 +2832,7 @@ static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh,
                 if (i != sh->pd_idx && i != sh->qd_idx) {
                         int dd_idx, j;
                         struct stripe_head *sh2;
+                        struct async_submit_ctl submit;
 
                         sector_t bn = compute_blocknr(sh, i, 1);
                         sector_t s = raid5_compute_sector(conf, bn, 0,
@@ -2685,9 +2852,10 @@ static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh,
                         }
 
                         /* place all the copies on one channel */
+                        init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
                         tx = async_memcpy(sh2->dev[dd_idx].page,
-                                sh->dev[i].page, 0, 0, STRIPE_SIZE,
-                                ASYNC_TX_DEP_ACK, tx, NULL, NULL);
+                                          sh->dev[i].page, 0, 0, STRIPE_SIZE,
+                                          &submit);
 
                         set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
                         set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
@@ -2756,7 +2924,8 @@ static bool handle_stripe5(struct stripe_head *sh)
         rcu_read_lock();
         for (i=disks; i--; ) {
                 mdk_rdev_t *rdev;
-                struct r5dev *dev = &sh->dev[i];
+
+                dev = &sh->dev[i];
                 clear_bit(R5_Insync, &dev->flags);
 
                 pr_debug("check %d: state 0x%lx toread %p read %p write %p "
@@ -2973,7 +3142,7 @@ static bool handle_stripe5(struct stripe_head *sh)
                 /* Need to write out all blocks after computing parity */
                 sh->disks = conf->raid_disks;
                 stripe_set_idx(sh->sector, conf, 0, sh);
-                schedule_reconstruction5(sh, &s, 1, 1);
+                schedule_reconstruction(sh, &s, 1, 1);
         } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
                 clear_bit(STRIPE_EXPAND_READY, &sh->state);
                 atomic_dec(&conf->reshape_stripes);
@@ -2993,7 +3162,7 @@ static bool handle_stripe5(struct stripe_head *sh)
                 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);
 
         if (s.ops_request)
-                raid5_run_ops(sh, s.ops_request);
+                raid_run_ops(sh, s.ops_request);
 
         ops_run_io(sh, &s);
 
@@ -3002,7 +3171,7 @@ static bool handle_stripe5(struct stripe_head *sh)
         return blocked_rdev == NULL;
 }
 
-static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
+static bool handle_stripe6(struct stripe_head *sh)
 {
         raid5_conf_t *conf = sh->raid_conf;
         int disks = sh->disks;
@@ -3014,9 +3183,10 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
         mdk_rdev_t *blocked_rdev = NULL;
 
         pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
-                "pd_idx=%d, qd_idx=%d\n",
+                "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
                (unsigned long long)sh->sector, sh->state,
-               atomic_read(&sh->count), pd_idx, qd_idx);
+               atomic_read(&sh->count), pd_idx, qd_idx,
+               sh->check_state, sh->reconstruct_state);
         memset(&s, 0, sizeof(s));
 
         spin_lock(&sh->lock);
@@ -3036,35 +3206,26 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
 
                 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
                         i, dev->flags, dev->toread, dev->towrite, dev->written);
-                /* maybe we can reply to a read */
-                if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread) {
-                        struct bio *rbi, *rbi2;
-                        pr_debug("Return read for disc %d\n", i);
-                        spin_lock_irq(&conf->device_lock);
-                        rbi = dev->toread;
-                        dev->toread = NULL;
-                        if (test_and_clear_bit(R5_Overlap, &dev->flags))
-                                wake_up(&conf->wait_for_overlap);
-                        spin_unlock_irq(&conf->device_lock);
-                        while (rbi && rbi->bi_sector < dev->sector + STRIPE_SECTORS) {
-                                copy_data(0, rbi, dev->page, dev->sector);
-                                rbi2 = r5_next_bio(rbi, dev->sector);
-                                spin_lock_irq(&conf->device_lock);
-                                if (!raid5_dec_bi_phys_segments(rbi)) {
-                                        rbi->bi_next = return_bi;
-                                        return_bi = rbi;
-                                }
-                                spin_unlock_irq(&conf->device_lock);
-                                rbi = rbi2;
-                        }
-                }
+                /* maybe we can reply to a read
+                 *
+                 * new wantfill requests are only permitted while
+                 * ops_complete_biofill is guaranteed to be inactive
+                 */
+                if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
+                    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
+                        set_bit(R5_Wantfill, &dev->flags);
 
                 /* now count some things */
                 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
                 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
+                if (test_bit(R5_Wantcompute, &dev->flags)) {
+                        s.compute++;
+                        BUG_ON(s.compute > 2);
+                }
 
-
-                if (dev->toread)
+                if (test_bit(R5_Wantfill, &dev->flags)) {
+                        s.to_fill++;
+                } else if (dev->toread)
                         s.to_read++;
                 if (dev->towrite) {
                         s.to_write++;
@@ -3105,6 +3266,11 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
                 blocked_rdev = NULL;
         }
 
+        if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
+                set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
+                set_bit(STRIPE_BIOFILL_RUN, &sh->state);
+        }
+
         pr_debug("locked=%d uptodate=%d to_read=%d"
                " to_write=%d failed=%d failed_num=%d,%d\n",
                s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
@@ -3145,19 +3311,62 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
          * or to load a block that is being partially written.
          */
         if (s.to_read || s.non_overwrite || (s.to_write && s.failed) ||
-            (s.syncing && (s.uptodate < disks)) || s.expanding)
+            (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
                 handle_stripe_fill6(sh, &s, &r6s, disks);
 
-        /* now to consider writing and what else, if anything should be read */
-        if (s.to_write)
+        /* Now we check to see if any write operations have recently
+         * completed
+         */
+        if (sh->reconstruct_state == reconstruct_state_drain_result) {
+                int qd_idx = sh->qd_idx;
+
+                sh->reconstruct_state = reconstruct_state_idle;
+                /* All the 'written' buffers and the parity blocks are ready to
+                 * be written back to disk
+                 */
+                BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
+                BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags));
+                for (i = disks; i--; ) {
+                        dev = &sh->dev[i];
+                        if (test_bit(R5_LOCKED, &dev->flags) &&
+                            (i == sh->pd_idx || i == qd_idx ||
+                             dev->written)) {
+                                pr_debug("Writing block %d\n", i);
+                                BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
+                                set_bit(R5_Wantwrite, &dev->flags);
+                                if (!test_bit(R5_Insync, &dev->flags) ||
+                                    ((i == sh->pd_idx || i == qd_idx) &&
+                                      s.failed == 0))
+                                        set_bit(STRIPE_INSYNC, &sh->state);
+                        }
+                }
+                if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
+                        atomic_dec(&conf->preread_active_stripes);
+                        if (atomic_read(&conf->preread_active_stripes) <
+                                IO_THRESHOLD)
+                                md_wakeup_thread(conf->mddev->thread);
+                }
+        }
+
+        /* Now to consider new write requests and what else, if anything
+         * should be read.  We do not handle new writes when:
+         * 1/ A 'write' operation (copy+gen_syndrome) is already in flight.
+         * 2/ A 'check' operation is in flight, as it may clobber the parity
+         *    block.
+         */
+        if (s.to_write && !sh->reconstruct_state && !sh->check_state)
                 handle_stripe_dirtying6(conf, sh, &s, &r6s, disks);
 
         /* maybe we need to check and possibly fix the parity for this stripe
          * Any reads will already have been scheduled, so we just see if enough
-         * data is available
+         * data is available.  The parity check is held off while parity
+         * dependent operations are in flight.
          */
-        if (s.syncing && s.locked == 0 && !test_bit(STRIPE_INSYNC, &sh->state))
-                handle_parity_checks6(conf, sh, &s, &r6s, tmp_page, disks);
+        if (sh->check_state ||
+            (s.syncing && s.locked == 0 &&
+             !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
+             !test_bit(STRIPE_INSYNC, &sh->state)))
+                handle_parity_checks6(conf, sh, &s, &r6s, disks);
 
         if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
                 md_done_sync(conf->mddev, STRIPE_SECTORS,1);
@@ -3178,15 +3387,29 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
                                         set_bit(R5_Wantwrite, &dev->flags);
                                         set_bit(R5_ReWrite, &dev->flags);
                                         set_bit(R5_LOCKED, &dev->flags);
+                                        s.locked++;
                                 } else {
                                         /* let's read it back */
                                         set_bit(R5_Wantread, &dev->flags);
                                         set_bit(R5_LOCKED, &dev->flags);
+                                        s.locked++;
                                 }
                         }
                 }
 
-        if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state)) {
+        /* Finish reconstruct operations initiated by the expansion process */
+        if (sh->reconstruct_state == reconstruct_state_result) {
+                sh->reconstruct_state = reconstruct_state_idle;
+                clear_bit(STRIPE_EXPANDING, &sh->state);
+                for (i = conf->raid_disks; i--; ) {
+                        set_bit(R5_Wantwrite, &sh->dev[i].flags);
+                        set_bit(R5_LOCKED, &sh->dev[i].flags);
+                        s.locked++;
+                }
+        }
+
+        if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
+            !sh->reconstruct_state) {
                 struct stripe_head *sh2
                         = get_active_stripe(conf, sh->sector, 1, 1, 1);
                 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
@@ -3207,14 +3430,8 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
                 /* Need to write out all blocks after computing P&Q */
                 sh->disks = conf->raid_disks;
                 stripe_set_idx(sh->sector, conf, 0, sh);
-                compute_parity6(sh, RECONSTRUCT_WRITE);
-                for (i = conf->raid_disks ; i-- ;  ) {
-                        set_bit(R5_LOCKED, &sh->dev[i].flags);
-                        s.locked++;
-                        set_bit(R5_Wantwrite, &sh->dev[i].flags);
-                }
-                clear_bit(STRIPE_EXPANDING, &sh->state);
-        } else if (s.expanded) {
+                schedule_reconstruction(sh, &s, 1, 1);
+        } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
                 clear_bit(STRIPE_EXPAND_READY, &sh->state);
                 atomic_dec(&conf->reshape_stripes);
                 wake_up(&conf->wait_for_overlap);
@@ -3232,6 +3449,9 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
         if (unlikely(blocked_rdev))
                 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);
 
+        if (s.ops_request)
+                raid_run_ops(sh, s.ops_request);
+
         ops_run_io(sh, &s);
 
         return_io(return_bi);
@@ -3240,16 +3460,14 @@ static bool handle_stripe6(struct stripe_head *sh, struct page *tmp_page)
 }
 
 /* returns true if the stripe was handled */
-static bool handle_stripe(struct stripe_head *sh, struct page *tmp_page)
+static bool handle_stripe(struct stripe_head *sh)
 {
         if (sh->raid_conf->level == 6)
-                return handle_stripe6(sh, tmp_page);
+                return handle_stripe6(sh);
         else
                 return handle_stripe5(sh);
 }
 
-
-
 static void raid5_activate_delayed(raid5_conf_t *conf)
 {
         if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
@@ -3331,6 +3549,9 @@ static int raid5_congested(void *data, int bits)
         /* No difference between reads and writes.  Just check
          * how busy the stripe_cache is
          */
+
+        if (mddev_congested(mddev, bits))
+                return 1;
         if (conf->inactive_blocked)
                 return 1;
         if (conf->quiesce)
@@ -3880,7 +4101,7 @@ static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped
         INIT_LIST_HEAD(&stripes);
         for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
                 int j;
-                int skipped = 0;
+                int skipped_disk = 0;
                 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
                 set_bit(STRIPE_EXPANDING, &sh->state);
                 atomic_inc(&conf->reshape_stripes);
@@ -3896,14 +4117,14 @@ static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped
                                 continue;
                         s = compute_blocknr(sh, j, 0);
                         if (s < raid5_size(mddev, 0, 0)) {
-                                skipped = 1;
+                                skipped_disk = 1;
                                 continue;
                         }
                         memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
                         set_bit(R5_Expanded, &sh->dev[j].flags);
                         set_bit(R5_UPTODATE, &sh->dev[j].flags);
                 }
-                if (!skipped) {
+                if (!skipped_disk) {
                         set_bit(STRIPE_EXPAND_READY, &sh->state);
                         set_bit(STRIPE_HANDLE, &sh->state);
                 }
@@ -4057,7 +4278,7 @@ static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *ski
         spin_unlock(&sh->lock);
 
         /* wait for any blocked device to be handled */
-        while(unlikely(!handle_stripe(sh, NULL)))
+        while (unlikely(!handle_stripe(sh)))
                 ;
         release_stripe(sh);
 
@@ -4114,7 +4335,7 @@ static int  retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio)
                         return handled;
                 }
 
-                handle_stripe(sh, NULL);
+                handle_stripe(sh);
                 release_stripe(sh);
                 handled++;
         }
@@ -4128,6 +4349,36 @@ static int  retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio)
         return handled;
 }
 
+#ifdef CONFIG_MULTICORE_RAID456
+static void __process_stripe(void *param, async_cookie_t cookie)
+{
+        struct stripe_head *sh = param;
+
+        handle_stripe(sh);
+        release_stripe(sh);
+}
+
+static void process_stripe(struct stripe_head *sh, struct list_head *domain)
+{
+        async_schedule_domain(__process_stripe, sh, domain);
+}
+
+static void synchronize_stripe_processing(struct list_head *domain)
+{
+        async_synchronize_full_domain(domain);
+}
+#else
+static void process_stripe(struct stripe_head *sh, struct list_head *domain)
+{
+        handle_stripe(sh);
+        release_stripe(sh);
+        cond_resched();
+}
+
+static void synchronize_stripe_processing(struct list_head *domain)
+{
+}
+#endif
 
 
 /*
@@ -4142,6 +4393,7 @@ static void raid5d(mddev_t *mddev)
         struct stripe_head *sh;
         raid5_conf_t *conf = mddev->private;
         int handled;
+        LIST_HEAD(raid_domain);
 
         pr_debug("+++ raid5d active\n");
 
@@ -4178,8 +4430,7 @@ static void raid5d(mddev_t *mddev)
                 spin_unlock_irq(&conf->device_lock);
                 
                 handled++;
-                handle_stripe(sh, conf->spare_page);
-                release_stripe(sh);
+                process_stripe(sh, &raid_domain);
 
                 spin_lock_irq(&conf->device_lock);
         }
@@ -4187,6 +4438,7 @@ static void raid5d(mddev_t *mddev)
 
         spin_unlock_irq(&conf->device_lock);
 
+        synchronize_stripe_processing(&raid_domain);
         async_tx_issue_pending_all();
         unplug_slaves(mddev);
 
@@ -4319,15 +4571,118 @@ raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks)
         return sectors * (raid_disks - conf->max_degraded);
 }
 
+static void raid5_free_percpu(raid5_conf_t *conf)
+{
+        struct raid5_percpu *percpu;
+        unsigned long cpu;
+
+        if (!conf->percpu)
+                return;
+
+        get_online_cpus();
+        for_each_possible_cpu(cpu) {
+                percpu = per_cpu_ptr(conf->percpu, cpu);
+                safe_put_page(percpu->spare_page);
+                kfree(percpu->scribble);
+        }
+#ifdef CONFIG_HOTPLUG_CPU
+        unregister_cpu_notifier(&conf->cpu_notify);
+#endif
+        put_online_cpus();
+
+        free_percpu(conf->percpu);
+}
+
 static void free_conf(raid5_conf_t *conf)
 {
         shrink_stripes(conf);
-        safe_put_page(conf->spare_page);
+        raid5_free_percpu(conf);
         kfree(conf->disks);
         kfree(conf->stripe_hashtbl);
         kfree(conf);
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
+                              void *hcpu)
+{
+        raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify);
+        long cpu = (long)hcpu;
+        struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
+
+        switch (action) {
+        case CPU_UP_PREPARE:
+        case CPU_UP_PREPARE_FROZEN:
+                if (conf->level == 6 && !percpu->spare_page)
+                        percpu->spare_page = alloc_page(GFP_KERNEL);
+                if (!percpu->scribble)
+                        percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
+
+                if (!percpu->scribble ||
+                    (conf->level == 6 && !percpu->spare_page)) {
+                        safe_put_page(percpu->spare_page);
+                        kfree(percpu->scribble);
+                        pr_err("%s: failed memory allocation for cpu%ld\n",
+                               __func__, cpu);
+                        return NOTIFY_BAD;
+                }
+                break;
+        case CPU_DEAD:
+        case CPU_DEAD_FROZEN:
+                safe_put_page(percpu->spare_page);
+                kfree(percpu->scribble);
+                percpu->spare_page = NULL;
+                percpu->scribble = NULL;
+                break;
+        default:
+                break;
+        }
+        return NOTIFY_OK;
+}
+#endif
+
+static int raid5_alloc_percpu(raid5_conf_t *conf)
+{
+        unsigned long cpu;
+        struct page *spare_page;
+        struct raid5_percpu *allcpus;
+        void *scribble;
+        int err;
+
+        allcpus = alloc_percpu(struct raid5_percpu);
+        if (!allcpus)
+                return -ENOMEM;
+        conf->percpu = allcpus;
+
+        get_online_cpus();
+        err = 0;
+        for_each_present_cpu(cpu) {
+                if (conf->level == 6) {
+                        spare_page = alloc_page(GFP_KERNEL);
+                        if (!spare_page) {
+                                err = -ENOMEM;
+                                break;
+                        }
+                        per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
+                }
+                scribble = kmalloc(scribble_len(conf->raid_disks), GFP_KERNEL);
+                if (!scribble) {
+                        err = -ENOMEM;
+                        break;
+                }
+                per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
+        }
+#ifdef CONFIG_HOTPLUG_CPU
+        conf->cpu_notify.notifier_call = raid456_cpu_notify;
+        conf->cpu_notify.priority = 0;
+        if (err == 0)
+                err = register_cpu_notifier(&conf->cpu_notify);
+#endif
+        put_online_cpus();
+
+        return err;
+}
+
 static raid5_conf_t *setup_conf(mddev_t *mddev)
 {
         raid5_conf_t *conf;
@@ -4369,6 +4724,7 @@ static raid5_conf_t *setup_conf(mddev_t *mddev)
                 goto abort;
 
         conf->raid_disks = mddev->raid_disks;
+        conf->scribble_len = scribble_len(conf->raid_disks);
         if (mddev->reshape_position == MaxSector)
                 conf->previous_raid_disks = mddev->raid_disks;
         else
@@ -4384,11 +4740,10 @@ static raid5_conf_t *setup_conf(mddev_t *mddev)
         if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
                 goto abort;
 
-        if (mddev->new_level == 6) {
-                conf->spare_page = alloc_page(GFP_KERNEL);
-                if (!conf->spare_page)
-                        goto abort;
-        }
+        conf->level = mddev->new_level;
+        if (raid5_alloc_percpu(conf) != 0)
+                goto abort;
+
         spin_lock_init(&conf->device_lock);
         init_waitqueue_head(&conf->wait_for_stripe);
         init_waitqueue_head(&conf->wait_for_overlap);
@@ -4447,7 +4802,7 @@ static raid5_conf_t *setup_conf(mddev_t *mddev)
                 printk(KERN_INFO "raid5: allocated %dkB for %s\n",
                         memory, mdname(mddev));
 
-        conf->thread = md_register_thread(raid5d, mddev, "%s_raid5");
+        conf->thread = md_register_thread(raid5d, mddev, NULL);
         if (!conf->thread) {
                 printk(KERN_ERR
                        "raid5: couldn't allocate thread for %s\n",
@@ -4613,7 +4968,7 @@ static int run(mddev_t *mddev)
                 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
                 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
                 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
-                                                        "%s_reshape");
+                                                        "reshape");
         }
 
         /* read-ahead size must cover two whole stripes, which is
@@ -5031,7 +5386,7 @@ static int raid5_start_reshape(mddev_t *mddev)
         set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
         set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
         mddev->sync_thread = md_register_thread(md_do_sync, mddev,
-                                                "%s_reshape");
+                                                "reshape");
         if (!mddev->sync_thread) {
                 mddev->recovery = 0;
                 spin_lock_irq(&conf->device_lock);
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 9459689c4ea0..2390e0e83daf 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -2,6 +2,7 @@
 #define _RAID5_H
 
 #include <linux/raid/xor.h>
+#include <linux/dmaengine.h>
 
 /*
  *
@@ -175,7 +176,9 @@
  */
 enum check_states {
         check_state_idle = 0,
-        check_state_run, /* parity check */
+        check_state_run, /* xor parity check */
+        check_state_run_q, /* q-parity check */
+        check_state_run_pq, /* pq dual parity check */
         check_state_check_result,
         check_state_compute_run, /* parity repair */
         check_state_compute_result,
@@ -215,8 +218,8 @@ struct stripe_head {
          * @target - STRIPE_OP_COMPUTE_BLK target
          */
         struct stripe_operations {
-                int                target;
-                u32                zero_sum_result;
+                int                  target, target2;
+                enum sum_check_flags zero_sum_result;
         } ops;
         struct r5dev {
                 struct bio      req;
@@ -298,7 +301,7 @@ struct r6_state {
 #define STRIPE_OP_COMPUTE_BLK   1
 #define STRIPE_OP_PREXOR        2
 #define STRIPE_OP_BIODRAIN      3
-#define STRIPE_OP_POSTXOR       4
+#define STRIPE_OP_RECONSTRUCT   4
 #define STRIPE_OP_CHECK 5
 
 /*
@@ -385,8 +388,21 @@ struct raid5_private_data {
                                             * (fresh device added).
                                             * Cleared when a sync completes.
                                             */
-
-        struct page             *spare_page; /* Used when checking P/Q in raid6 */
+        /* per cpu variables */
+        struct raid5_percpu {
+                struct page     *spare_page; /* Used when checking P/Q in raid6 */
+                void            *scribble;   /* space for constructing buffer
+                                              * lists and performing address
+                                              * conversions
+                                              */
+        } *percpu;
+        size_t                  scribble_len; /* size of scribble region must be
+                                               * associated with conf to handle
+                                               * cpu hotplug while reshaping
+                                               */
+#ifdef CONFIG_HOTPLUG_CPU
+        struct notifier_block   cpu_notify;
+#endif
 
         /*
          * Free stripes pool
diff --git a/drivers/media/dvb/dvb-core/dvbdev.h b/drivers/media/dvb/dvb-core/dvbdev.h
index 895e2efca8a9..01fc70484743 100644
--- a/drivers/media/dvb/dvb-core/dvbdev.h
+++ b/drivers/media/dvb/dvb-core/dvbdev.h
@@ -31,10 +31,9 @@
 #define DVB_MAJOR 212
 
 #if defined(CONFIG_DVB_MAX_ADAPTERS) && CONFIG_DVB_MAX_ADAPTERS > 0
-#define DVB_MAX_ADAPTERS CONFIG_DVB_MAX_ADAPTERS
+  #define DVB_MAX_ADAPTERS CONFIG_DVB_MAX_ADAPTERS
 #else
-#warning invalid CONFIG_DVB_MAX_ADAPTERS value
-#define DVB_MAX_ADAPTERS 8
+  #define DVB_MAX_ADAPTERS 8
 #endif
 
 #define DVB_UNSET (-1)
diff --git a/drivers/media/dvb/dvb-usb/Kconfig b/drivers/media/dvb/dvb-usb/Kconfig
index 0e4b97fba384..9744b0692417 100644
--- a/drivers/media/dvb/dvb-usb/Kconfig
+++ b/drivers/media/dvb/dvb-usb/Kconfig
@@ -75,7 +75,7 @@ config DVB_USB_DIB0700
         select DVB_DIB3000MC if !DVB_FE_CUSTOMISE
         select DVB_S5H1411 if !DVB_FE_CUSTOMISE
         select DVB_LGDT3305 if !DVB_FE_CUSTOMISE
-        select DVB_TUNER_DIB0070 if !DVB_FE_CUSTOMISE
+        select DVB_TUNER_DIB0070
         select MEDIA_TUNER_MT2060 if !MEDIA_TUNER_CUSTOMISE
         select MEDIA_TUNER_MT2266 if !MEDIA_TUNER_CUSTOMISE
         select MEDIA_TUNER_XC2028 if !MEDIA_TUNER_CUSTOMISE
diff --git a/drivers/media/video/saa7164/saa7164-api.c b/drivers/media/video/saa7164/saa7164-api.c
index bb6df1b276be..6f094a96ac81 100644
--- a/drivers/media/video/saa7164/saa7164-api.c
+++ b/drivers/media/video/saa7164/saa7164-api.c
@@ -415,7 +415,7 @@ int saa7164_api_enum_subdevs(struct saa7164_dev *dev)
                 goto out;
         }
 
-        if (debug & DBGLVL_API)
+        if (saa_debug & DBGLVL_API)
                 saa7164_dumphex16(dev, buf, (buflen/16)*16);
 
         saa7164_api_dump_subdevs(dev, buf, buflen);
@@ -480,7 +480,7 @@ int saa7164_api_i2c_read(struct saa7164_i2c *bus, u8 addr, u32 reglen, u8 *reg,
 
         dprintk(DBGLVL_API, "%s() len = %d bytes\n", __func__, len);
 
-        if (debug & DBGLVL_I2C)
+        if (saa_debug & DBGLVL_I2C)
                 saa7164_dumphex16(dev, buf, 2 * 16);
 
         ret = saa7164_cmd_send(bus->dev, unitid, GET_CUR,
@@ -488,7 +488,7 @@ int saa7164_api_i2c_read(struct saa7164_i2c *bus, u8 addr, u32 reglen, u8 *reg,
         if (ret != SAA_OK)
                 printk(KERN_ERR "%s() error, ret(2) = 0x%x\n", __func__, ret);
         else {
-                if (debug & DBGLVL_I2C)
+                if (saa_debug & DBGLVL_I2C)
                         saa7164_dumphex16(dev, buf, sizeof(buf));
                 memcpy(data, (buf + 2 * sizeof(u32) + reglen), datalen);
         }
@@ -548,7 +548,7 @@ int saa7164_api_i2c_write(struct saa7164_i2c *bus, u8 addr, u32 datalen,
         *((u32 *)(buf + 1 * sizeof(u32))) = datalen - reglen;
         memcpy((buf + 2 * sizeof(u32)), data, datalen);
 
-        if (debug & DBGLVL_I2C)
+        if (saa_debug & DBGLVL_I2C)
                 saa7164_dumphex16(dev, buf, sizeof(buf));
 
         ret = saa7164_cmd_send(bus->dev, unitid, SET_CUR,
diff --git a/drivers/media/video/saa7164/saa7164-cmd.c b/drivers/media/video/saa7164/saa7164-cmd.c
index e097f1a0969a..c45966edc0cf 100644
--- a/drivers/media/video/saa7164/saa7164-cmd.c
+++ b/drivers/media/video/saa7164/saa7164-cmd.c
@@ -250,7 +250,7 @@ int saa7164_cmd_wait(struct saa7164_dev *dev, u8 seqno)
         unsigned long stamp;
         int r;
 
-        if (debug >= 4)
+        if (saa_debug >= 4)
                 saa7164_bus_dump(dev);
 
         dprintk(DBGLVL_CMD, "%s(seqno=%d)\n", __func__, seqno);
diff --git a/drivers/media/video/saa7164/saa7164-core.c b/drivers/media/video/saa7164/saa7164-core.c
index f0dbead188c8..709affc31042 100644
--- a/drivers/media/video/saa7164/saa7164-core.c
+++ b/drivers/media/video/saa7164/saa7164-core.c
@@ -45,8 +45,8 @@ MODULE_LICENSE("GPL");
  32 bus
  */
 
-unsigned int debug;
-module_param(debug, int, 0644);
+unsigned int saa_debug;
+module_param_named(debug, saa_debug, int, 0644);
 MODULE_PARM_DESC(debug, "enable debug messages");
 
 unsigned int waitsecs = 10;
@@ -653,7 +653,7 @@ static int __devinit saa7164_initdev(struct pci_dev *pci_dev,
                 printk(KERN_ERR "%s() Unsupported board detected, "
                         "registering without firmware\n", __func__);
 
-        dprintk(1, "%s() parameter debug = %d\n", __func__, debug);
+        dprintk(1, "%s() parameter debug = %d\n", __func__, saa_debug);
         dprintk(1, "%s() parameter waitsecs = %d\n", __func__, waitsecs);
 
 fail_fw:
diff --git a/drivers/media/video/saa7164/saa7164.h b/drivers/media/video/saa7164/saa7164.h
index 6753008a9c9b..42660b546f0e 100644
--- a/drivers/media/video/saa7164/saa7164.h
+++ b/drivers/media/video/saa7164/saa7164.h
@@ -375,9 +375,9 @@ extern int saa7164_buffer_dealloc(struct saa7164_tsport *port,
 
 /* ----------------------------------------------------------- */
 
-extern unsigned int debug;
+extern unsigned int saa_debug;
 #define dprintk(level, fmt, arg...)\
-        do { if (debug & level)\
+        do { if (saa_debug & level)\
                 printk(KERN_DEBUG "%s: " fmt, dev->name, ## arg);\
         } while (0)
 
diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c
index a5b448ea4eab..b3bf1c44d74d 100644
--- a/drivers/memstick/core/memstick.c
+++ b/drivers/memstick/core/memstick.c
@@ -339,9 +339,9 @@ static int h_memstick_read_dev_id(struct memstick_dev *card,
                         card->id.type = id_reg.type;
                         card->id.category = id_reg.category;
                         card->id.class = id_reg.class;
+                        dev_dbg(&card->dev, "if_mode = %02x\n", id_reg.if_mode);
                 }
                 complete(&card->mrq_complete);
-                dev_dbg(&card->dev, "if_mode = %02x\n", id_reg.if_mode);
                 return -EAGAIN;
         }
 }
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 79689b10f937..766e21e15574 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -937,6 +937,8 @@ static int quicktest1(unsigned long arg)
 
         /* Need  1K cacheline aligned that does not cross page boundary */
         p = kmalloc(4096, 0);
+        if (p == NULL)
+                return -ENOMEM;
         mq = ALIGNUP(p, 1024);
         memset(mes, 0xee, sizeof(mes));
         dw = mq;
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index 9cbf95bedce6..ccd4408a26c7 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -340,10 +340,9 @@ static struct proc_dir_entry *proc_gru __read_mostly;
 
 static int create_proc_file(struct proc_entry *p)
 {
-        p->entry = create_proc_entry(p->name, p->mode, proc_gru);
+        p->entry = proc_create(p->name, p->mode, proc_gru, p->fops);
         if (!p->entry)
                 return -1;
-        p->entry->proc_fops = p->fops;
         return 0;
 }
 
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index 065fa818be57..fc25586b7ee1 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -599,6 +599,7 @@ atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
         struct scatterlist              *sg;
         unsigned int                    i;
         enum dma_data_direction         direction;
+        unsigned int                    sglen;
 
         /*
          * We don't do DMA on "complex" transfers, i.e. with
@@ -628,11 +629,14 @@ atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
         else
                 direction = DMA_TO_DEVICE;
 
+        sglen = dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, direction);
+        if (sglen != data->sg_len)
+                goto unmap_exit;
         desc = chan->device->device_prep_slave_sg(chan,
                         data->sg, data->sg_len, direction,
                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
         if (!desc)
-                return -ENOMEM;
+                goto unmap_exit;
 
         host->dma.data_desc = desc;
         desc->callback = atmci_dma_complete;
@@ -643,6 +647,9 @@ atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
         chan->device->device_issue_pending(chan);
 
         return 0;
+unmap_exit:
+        dma_unmap_sg(&host->pdev->dev, data->sg, sglen, direction);
+        return -ENOMEM;
 }
 
 #else /* CONFIG_MMC_ATMELMCI_DMA */
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index 8741d0f5146a..3d1e5329da12 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -22,12 +22,13 @@
 #include <linux/clk.h>
 #include <linux/scatterlist.h>
 #include <linux/gpio.h>
+#include <linux/amba/mmci.h>
+#include <linux/regulator/consumer.h>
 
 #include <asm/cacheflush.h>
 #include <asm/div64.h>
 #include <asm/io.h>
 #include <asm/sizes.h>
-#include <asm/mach/mmc.h>
 
 #include "mmci.h"
 
@@ -38,6 +39,36 @@
 
 static unsigned int fmax = 515633;
 
+/*
+ * This must be called with host->lock held
+ */
+static void mmci_set_clkreg(struct mmci_host *host, unsigned int desired)
+{
+        u32 clk = 0;
+
+        if (desired) {
+                if (desired >= host->mclk) {
+                        clk = MCI_CLK_BYPASS;
+                        host->cclk = host->mclk;
+                } else {
+                        clk = host->mclk / (2 * desired) - 1;
+                        if (clk >= 256)
+                                clk = 255;
+                        host->cclk = host->mclk / (2 * (clk + 1));
+                }
+                if (host->hw_designer == 0x80)
+                        clk |= MCI_FCEN; /* Bug fix in ST IP block */
+                clk |= MCI_CLK_ENABLE;
+                /* This hasn't proven to be worthwhile */
+                /* clk |= MCI_CLK_PWRSAVE; */
+        }
+
+        if (host->mmc->ios.bus_width == MMC_BUS_WIDTH_4)
+                clk |= MCI_WIDE_BUS;
+
+        writel(clk, host->base + MMCICLOCK);
+}
+
 static void
 mmci_request_end(struct mmci_host *host, struct mmc_request *mrq)
 {
@@ -419,30 +450,31 @@ static void mmci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 static void mmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 {
         struct mmci_host *host = mmc_priv(mmc);
-        u32 clk = 0, pwr = 0;
-
-        if (ios->clock) {
-                if (ios->clock >= host->mclk) {
-                        clk = MCI_CLK_BYPASS;
-                        host->cclk = host->mclk;
-                } else {
-                        clk = host->mclk / (2 * ios->clock) - 1;
-                        if (clk >= 256)
-                                clk = 255;
-                        host->cclk = host->mclk / (2 * (clk + 1));
-                }
-                if (host->hw_designer == AMBA_VENDOR_ST)
-                        clk |= MCI_FCEN; /* Bug fix in ST IP block */
-                clk |= MCI_CLK_ENABLE;
-        }
-
-        if (host->plat->translate_vdd)
-                pwr |= host->plat->translate_vdd(mmc_dev(mmc), ios->vdd);
+        u32 pwr = 0;
+        unsigned long flags;
 
         switch (ios->power_mode) {
         case MMC_POWER_OFF:
+                if(host->vcc &&
+                   regulator_is_enabled(host->vcc))
+                        regulator_disable(host->vcc);
                 break;
         case MMC_POWER_UP:
+#ifdef CONFIG_REGULATOR
+                if (host->vcc)
+                        /* This implicitly enables the regulator */
+                        mmc_regulator_set_ocr(host->vcc, ios->vdd);
+#endif
+                /*
+                 * The translate_vdd function is not used if you have
+                 * an external regulator, or your design is really weird.
+                 * Using it would mean sending in power control BOTH using
+                 * a regulator AND the 4 MMCIPWR bits. If we don't have
+                 * a regulator, we might have some other platform specific
+                 * power control behind this translate function.
+                 */
+                if (!host->vcc && host->plat->translate_vdd)
+                        pwr |= host->plat->translate_vdd(mmc_dev(mmc), ios->vdd);
                 /* The ST version does not have this, fall through to POWER_ON */
                 if (host->hw_designer != AMBA_VENDOR_ST) {
                         pwr |= MCI_PWR_UP;
@@ -465,12 +497,16 @@ static void mmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
                 }
         }
 
-        writel(clk, host->base + MMCICLOCK);
+        spin_lock_irqsave(&host->lock, flags);
+
+        mmci_set_clkreg(host, ios->clock);
 
         if (host->pwr != pwr) {
                 host->pwr = pwr;
                 writel(pwr, host->base + MMCIPOWER);
         }
+
+        spin_unlock_irqrestore(&host->lock, flags);
 }
 
 static int mmci_get_ro(struct mmc_host *mmc)
@@ -517,7 +553,7 @@ static void mmci_check_status(unsigned long data)
 
 static int __devinit mmci_probe(struct amba_device *dev, struct amba_id *id)
 {
-        struct mmc_platform_data *plat = dev->dev.platform_data;
+        struct mmci_platform_data *plat = dev->dev.platform_data;
         struct mmci_host *host;
         struct mmc_host *mmc;
         int ret;
@@ -583,7 +619,30 @@ static int __devinit mmci_probe(struct amba_device *dev, struct amba_id *id)
         mmc->ops = &mmci_ops;
         mmc->f_min = (host->mclk + 511) / 512;
         mmc->f_max = min(host->mclk, fmax);
-        mmc->ocr_avail = plat->ocr_mask;
+#ifdef CONFIG_REGULATOR
+        /* If we're using the regulator framework, try to fetch a regulator */
+        host->vcc = regulator_get(&dev->dev, "vmmc");
+        if (IS_ERR(host->vcc))
+                host->vcc = NULL;
+        else {
+                int mask = mmc_regulator_get_ocrmask(host->vcc);
+
+                if (mask < 0)
+                        dev_err(&dev->dev, "error getting OCR mask (%d)\n",
+                                mask);
+                else {
+                        host->mmc->ocr_avail = (u32) mask;
+                        if (plat->ocr_mask)
+                                dev_warn(&dev->dev,
+                                 "Provided ocr_mask/setpower will not be used "
+                                 "(using regulator instead)\n");
+                }
+        }
+#endif
+        /* Fall back to platform data if no regulator is found */
+        if (host->vcc == NULL)
+                mmc->ocr_avail = plat->ocr_mask;
+        mmc->caps = plat->capabilities;
 
         /*
          * We can do SGIO
@@ -720,6 +779,10 @@ static int __devexit mmci_remove(struct amba_device *dev)
                 clk_disable(host->clk);
                 clk_put(host->clk);
 
+                if (regulator_is_enabled(host->vcc))
+                        regulator_disable(host->vcc);
+                regulator_put(host->vcc);
+
                 mmc_free_host(mmc);
 
                 amba_release_regions(dev);
diff --git a/drivers/mmc/host/mmci.h b/drivers/mmc/host/mmci.h
index 839f264c9725..1ceb9a90f59b 100644
--- a/drivers/mmc/host/mmci.h
+++ b/drivers/mmc/host/mmci.h
@@ -161,7 +161,7 @@ struct mmci_host {
         unsigned int            mclk;
         unsigned int            cclk;
         u32                     pwr;
-        struct mmc_platform_data *plat;
+        struct mmci_platform_data *plat;
 
         u8                      hw_designer;
         u8                      hw_revision:4;
@@ -175,6 +175,7 @@ struct mmci_host {
         struct scatterlist      *sg_ptr;
         unsigned int            sg_off;
         unsigned int            size;
+        struct regulator        *vcc;
 };
 
 static inline void mmci_init_sg(struct mmci_host *host, struct mmc_data *data)
diff --git a/drivers/mmc/host/pxamci.c b/drivers/mmc/host/pxamci.c
index e55ac792d68c..5e0b1529964d 100644
--- a/drivers/mmc/host/pxamci.c
+++ b/drivers/mmc/host/pxamci.c
@@ -28,6 +28,7 @@
 #include <linux/mmc/host.h>
 #include <linux/io.h>
 #include <linux/regulator/consumer.h>
+#include <linux/gpio.h>
 
 #include <asm/sizes.h>
 
@@ -96,10 +97,18 @@ static inline void pxamci_init_ocr(struct pxamci_host *host)
 
 static inline void pxamci_set_power(struct pxamci_host *host, unsigned int vdd)
 {
+        int on;
+
 #ifdef CONFIG_REGULATOR
         if (host->vcc)
                 mmc_regulator_set_ocr(host->vcc, vdd);
 #endif
+        if (!host->vcc && host->pdata &&
+            gpio_is_valid(host->pdata->gpio_power)) {
+                on = ((1 << vdd) & host->pdata->ocr_mask);
+                gpio_set_value(host->pdata->gpio_power,
+                               !!on ^ host->pdata->gpio_power_invert);
+        }
         if (!host->vcc && host->pdata && host->pdata->setpower)
                 host->pdata->setpower(mmc_dev(host->mmc), vdd);
 }
@@ -421,6 +430,12 @@ static int pxamci_get_ro(struct mmc_host *mmc)
 {
         struct pxamci_host *host = mmc_priv(mmc);
 
+        if (host->pdata && gpio_is_valid(host->pdata->gpio_card_ro)) {
+                if (host->pdata->gpio_card_ro_invert)
+                        return !gpio_get_value(host->pdata->gpio_card_ro);
+                else
+                        return gpio_get_value(host->pdata->gpio_card_ro);
+        }
         if (host->pdata && host->pdata->get_ro)
                 return !!host->pdata->get_ro(mmc_dev(mmc));
         /*
@@ -534,7 +549,7 @@ static int pxamci_probe(struct platform_device *pdev)
         struct mmc_host *mmc;
         struct pxamci_host *host = NULL;
         struct resource *r, *dmarx, *dmatx;
-        int ret, irq;
+        int ret, irq, gpio_cd = -1, gpio_ro = -1, gpio_power = -1;
 
         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         irq = platform_get_irq(pdev, 0);
@@ -661,13 +676,63 @@ static int pxamci_probe(struct platform_device *pdev)
         }
         host->dma_drcmrtx = dmatx->start;
 
+        if (host->pdata) {
+                gpio_cd = host->pdata->gpio_card_detect;
+                gpio_ro = host->pdata->gpio_card_ro;
+                gpio_power = host->pdata->gpio_power;
+        }
+        if (gpio_is_valid(gpio_power)) {
+                ret = gpio_request(gpio_power, "mmc card power");
+                if (ret) {
+                        dev_err(&pdev->dev, "Failed requesting gpio_power %d\n", gpio_power);
+                        goto out;
+                }
+                gpio_direction_output(gpio_power,
+                                      host->pdata->gpio_power_invert);
+        }
+        if (gpio_is_valid(gpio_ro)) {
+                ret = gpio_request(gpio_ro, "mmc card read only");
+                if (ret) {
+                        dev_err(&pdev->dev, "Failed requesting gpio_ro %d\n", gpio_power);
+                        goto err_gpio_ro;
+                }
+                gpio_direction_input(gpio_ro);
+        }
+        if (gpio_is_valid(gpio_cd)) {
+                ret = gpio_request(gpio_cd, "mmc card detect");
+                if (ret) {
+                        dev_err(&pdev->dev, "Failed requesting gpio_cd %d\n", gpio_power);
+                        goto err_gpio_cd;
+                }
+                gpio_direction_input(gpio_cd);
+
+                ret = request_irq(gpio_to_irq(gpio_cd), pxamci_detect_irq,
+                                  IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
+                                  "mmc card detect", mmc);
+                if (ret) {
+                        dev_err(&pdev->dev, "failed to request card detect IRQ\n");
+                        goto err_request_irq;
+                }
+        }
+
         if (host->pdata && host->pdata->init)
                 host->pdata->init(&pdev->dev, pxamci_detect_irq, mmc);
 
+        if (gpio_is_valid(gpio_power) && host->pdata->setpower)
+                dev_warn(&pdev->dev, "gpio_power and setpower() both defined\n");
+        if (gpio_is_valid(gpio_ro) && host->pdata->get_ro)
+                dev_warn(&pdev->dev, "gpio_ro and get_ro() both defined\n");
+
         mmc_add_host(mmc);
 
         return 0;
 
+err_request_irq:
+        gpio_free(gpio_cd);
+err_gpio_cd:
+        gpio_free(gpio_ro);
+err_gpio_ro:
+        gpio_free(gpio_power);
  out:
         if (host) {
                 if (host->dma >= 0)
@@ -688,12 +753,26 @@ static int pxamci_probe(struct platform_device *pdev)
 static int pxamci_remove(struct platform_device *pdev)
 {
         struct mmc_host *mmc = platform_get_drvdata(pdev);
+        int gpio_cd = -1, gpio_ro = -1, gpio_power = -1;
 
         platform_set_drvdata(pdev, NULL);
 
         if (mmc) {
                 struct pxamci_host *host = mmc_priv(mmc);
 
+                if (host->pdata) {
+                        gpio_cd = host->pdata->gpio_card_detect;
+                        gpio_ro = host->pdata->gpio_card_ro;
+                        gpio_power = host->pdata->gpio_power;
+                }
+                if (gpio_is_valid(gpio_cd)) {
+                        free_irq(gpio_to_irq(gpio_cd), mmc);
+                        gpio_free(gpio_cd);
+                }
+                if (gpio_is_valid(gpio_ro))
+                        gpio_free(gpio_ro);
+                if (gpio_is_valid(gpio_power))
+                        gpio_free(gpio_power);
                 if (host->vcc)
                         regulator_put(host->vcc);
 
@@ -725,20 +804,20 @@ static int pxamci_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM
-static int pxamci_suspend(struct platform_device *dev, pm_message_t state)
+static int pxamci_suspend(struct device *dev)
 {
-        struct mmc_host *mmc = platform_get_drvdata(dev);
+        struct mmc_host *mmc = dev_get_drvdata(dev);
         int ret = 0;
 
         if (mmc)
-                ret = mmc_suspend_host(mmc, state);
+                ret = mmc_suspend_host(mmc, PMSG_SUSPEND);
 
         return ret;
 }
 
-static int pxamci_resume(struct platform_device *dev)
+static int pxamci_resume(struct device *dev)
 {
-        struct mmc_host *mmc = platform_get_drvdata(dev);
+        struct mmc_host *mmc = dev_get_drvdata(dev);
         int ret = 0;
 
         if (mmc)
@@ -746,19 +825,22 @@ static int pxamci_resume(struct platform_device *dev)
 
         return ret;
 }
-#else
-#define pxamci_suspend  NULL
-#define pxamci_resume   NULL
+
+static struct dev_pm_ops pxamci_pm_ops = {
+        .suspend        = pxamci_suspend,
+        .resume         = pxamci_resume,
+};
 #endif
 
 static struct platform_driver pxamci_driver = {
         .probe          = pxamci_probe,
         .remove         = pxamci_remove,
-        .suspend        = pxamci_suspend,
-        .resume         = pxamci_resume,
         .driver         = {
                 .name   = DRIVER_NAME,
                 .owner  = THIS_MODULE,
+#ifdef CONFIG_PM
+                .pm     = &pxamci_pm_ops,
+#endif
         },
 };
 
diff --git a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig
index e4ec3659759a..ecf90f5c97c2 100644
--- a/drivers/mtd/Kconfig
+++ b/drivers/mtd/Kconfig
@@ -159,7 +159,7 @@ config MTD_AFS_PARTS
 
 config MTD_OF_PARTS
         tristate "Flash partition map based on OF description"
-        depends on PPC_OF && MTD_PARTITIONS
+        depends on (MICROBLAZE || PPC_OF) && MTD_PARTITIONS
         help
           This provides a partition parsing function which derives
           the partition map from the children of the flash node,
diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
index 3a9a960644b6..841e085ab74a 100644
--- a/drivers/mtd/maps/Kconfig
+++ b/drivers/mtd/maps/Kconfig
@@ -74,7 +74,7 @@ config MTD_PHYSMAP_BANKWIDTH
 
 config MTD_PHYSMAP_OF
         tristate "Flash device in physical memory map based on OF description"
-        depends on PPC_OF && (MTD_CFI || MTD_JEDECPROBE || MTD_ROM)
+        depends on (MICROBLAZE || PPC_OF) && (MTD_CFI || MTD_JEDECPROBE || MTD_ROM)
         help
           This provides a 'mapping' driver which allows the NOR Flash and
           ROM driver code to communicate with chips which are mapped
diff --git a/drivers/net/cris/eth_v10.c b/drivers/net/cris/eth_v10.c
index 15c0195ebd31..a24be34a3f7a 100644
--- a/drivers/net/cris/eth_v10.c
+++ b/drivers/net/cris/eth_v10.c
@@ -768,10 +768,24 @@ e100_negotiate(struct net_device* dev)
 
         e100_set_mdio_reg(dev, np->mii_if.phy_id, MII_ADVERTISE, data);
 
-        /* Renegotiate with link partner */
+        data = e100_get_mdio_reg(dev, np->mii_if.phy_id, MII_BMCR);
         if (autoneg_normal) {
-          data = e100_get_mdio_reg(dev, np->mii_if.phy_id, MII_BMCR);
-        data |= BMCR_ANENABLE | BMCR_ANRESTART;
+                /* Renegotiate with link partner */
+                data |= BMCR_ANENABLE | BMCR_ANRESTART;
+        } else {
+                /* Don't negotiate speed or duplex */
+                data &= ~(BMCR_ANENABLE | BMCR_ANRESTART);
+
+                /* Set speed and duplex static */
+                if (current_speed_selection == 10)
+                        data &= ~BMCR_SPEED100;
+                else
+                        data |= BMCR_SPEED100;
+
+                if (current_duplex != full)
+                        data &= ~BMCR_FULLDPLX;
+                else
+                        data |= BMCR_FULLDPLX;
         }
         e100_set_mdio_reg(dev, np->mii_if.phy_id, MII_BMCR, data);
 }
diff --git a/drivers/net/irda/pxaficp_ir.c b/drivers/net/irda/pxaficp_ir.c
index 1445e5865196..84db145d2b59 100644
--- a/drivers/net/irda/pxaficp_ir.c
+++ b/drivers/net/irda/pxaficp_ir.c
@@ -17,6 +17,7 @@
 #include <linux/etherdevice.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
+#include <linux/gpio.h>
 
 #include <net/irda/irda.h>
 #include <net/irda/irmod.h>
@@ -163,6 +164,22 @@ inline static void pxa_irda_fir_dma_tx_start(struct pxa_irda *si)
 }
 
 /*
+ * Set the IrDA communications mode.
+ */
+static void pxa_irda_set_mode(struct pxa_irda *si, int mode)
+{
+        if (si->pdata->transceiver_mode)
+                si->pdata->transceiver_mode(si->dev, mode);
+        else {
+                if (gpio_is_valid(si->pdata->gpio_pwdown))
+                        gpio_set_value(si->pdata->gpio_pwdown,
+                                        !(mode & IR_OFF) ^
+                                        !si->pdata->gpio_pwdown_inverted);
+                pxa2xx_transceiver_mode(si->dev, mode);
+        }
+}
+
+/*
  * Set the IrDA communications speed.
  */
 static int pxa_irda_set_speed(struct pxa_irda *si, int speed)
@@ -188,7 +205,7 @@ static int pxa_irda_set_speed(struct pxa_irda *si, int speed)
                         pxa_irda_disable_clk(si);
 
                         /* set board transceiver to SIR mode */
-                        si->pdata->transceiver_mode(si->dev, IR_SIRMODE);
+                        pxa_irda_set_mode(si, IR_SIRMODE);
 
                         /* enable the STUART clock */
                         pxa_irda_enable_sirclk(si);
@@ -222,7 +239,7 @@ static int pxa_irda_set_speed(struct pxa_irda *si, int speed)
                 ICCR0 = 0;
 
                 /* set board transceiver to FIR mode */
-                si->pdata->transceiver_mode(si->dev, IR_FIRMODE);
+                pxa_irda_set_mode(si, IR_FIRMODE);
 
                 /* enable the FICP clock */
                 pxa_irda_enable_firclk(si);
@@ -641,7 +658,7 @@ static void pxa_irda_shutdown(struct pxa_irda *si)
         local_irq_restore(flags);
 
         /* power off board transceiver */
-        si->pdata->transceiver_mode(si->dev, IR_OFF);
+        pxa_irda_set_mode(si, IR_OFF);
 
         printk(KERN_DEBUG "pxa_ir: irda shutdown\n");
 }
@@ -849,10 +866,26 @@ static int pxa_irda_probe(struct platform_device *pdev)
         if (err)
                 goto err_mem_5;
 
-        if (si->pdata->startup)
+        if (gpio_is_valid(si->pdata->gpio_pwdown)) {
+                err = gpio_request(si->pdata->gpio_pwdown, "IrDA switch");
+                if (err)
+                        goto err_startup;
+                err = gpio_direction_output(si->pdata->gpio_pwdown,
+                                        !si->pdata->gpio_pwdown_inverted);
+                if (err) {
+                        gpio_free(si->pdata->gpio_pwdown);
+                        goto err_startup;
+                }
+        }
+
+        if (si->pdata->startup) {
                 err = si->pdata->startup(si->dev);
-        if (err)
-                goto err_startup;
+                if (err)
+                        goto err_startup;
+        }
+
+        if (gpio_is_valid(si->pdata->gpio_pwdown) && si->pdata->startup)
+                dev_warn(si->dev, "gpio_pwdown and startup() both defined!\n");
 
         dev->netdev_ops = &pxa_irda_netdev_ops;
 
@@ -903,6 +936,8 @@ static int pxa_irda_remove(struct platform_device *_dev)
         if (dev) {
                 struct pxa_irda *si = netdev_priv(dev);
                 unregister_netdev(dev);
+                if (gpio_is_valid(si->pdata->gpio_pwdown))
+                        gpio_free(si->pdata->gpio_pwdown);
                 if (si->pdata->shutdown)
                         si->pdata->shutdown(si->dev);
                 kfree(si->tx_buff.head);
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index cee199ceba2f..3c16602172fc 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -33,6 +33,7 @@
  */
 
 #include <linux/mlx4/cmd.h>
+#include <linux/cache.h>
 
 #include "fw.h"
 #include "icm.h"
@@ -698,6 +699,7 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
 #define INIT_HCA_IN_SIZE                 0x200
 #define INIT_HCA_VERSION_OFFSET          0x000
 #define  INIT_HCA_VERSION                2
+#define INIT_HCA_CACHELINE_SZ_OFFSET     0x0e
 #define INIT_HCA_FLAGS_OFFSET            0x014
 #define INIT_HCA_QPC_OFFSET              0x020
 #define  INIT_HCA_QPC_BASE_OFFSET        (INIT_HCA_QPC_OFFSET + 0x10)
@@ -735,6 +737,9 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
 
         *((u8 *) mailbox->buf + INIT_HCA_VERSION_OFFSET) = INIT_HCA_VERSION;
 
+        *((u8 *) mailbox->buf + INIT_HCA_CACHELINE_SZ_OFFSET) =
+                (ilog2(cache_line_size()) - 4) << 5;
+
 #if defined(__LITTLE_ENDIAN)
         *(inbox + INIT_HCA_FLAGS_OFFSET / 4) &= ~cpu_to_be32(1 << 1);
 #elif defined(__BIG_ENDIAN)
diff --git a/drivers/net/wireless/arlan-proc.c b/drivers/net/wireless/arlan-proc.c
index 2ab1d59870f4..a8b689635a3b 100644
--- a/drivers/net/wireless/arlan-proc.c
+++ b/drivers/net/wireless/arlan-proc.c
@@ -402,7 +402,7 @@ static int arlan_setup_card_by_book(struct net_device *dev)
 
 static char arlan_drive_info[ARLAN_STR_SIZE] = "A655\n\0";
 
-static int arlan_sysctl_info(ctl_table * ctl, int write, struct file *filp,
+static int arlan_sysctl_info(ctl_table * ctl, int write,
                       void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int i;
@@ -629,7 +629,7 @@ final:
         *lenp = pos;
 
         if (!write)
-                retv = proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+                retv = proc_dostring(ctl, write, buffer, lenp, ppos);
         else
         {
                 *lenp = 0;
@@ -639,7 +639,7 @@ final:
 }
 
 
-static int arlan_sysctl_info161719(ctl_table * ctl, int write, struct file *filp,
+static int arlan_sysctl_info161719(ctl_table * ctl, int write,
                             void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int i;
@@ -669,11 +669,11 @@ static int arlan_sysctl_info161719(ctl_table * ctl, int write, struct file *filp
 
 final:
         *lenp = pos;
-        retv = proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        retv = proc_dostring(ctl, write, buffer, lenp, ppos);
         return retv;
 }
 
-static int arlan_sysctl_infotxRing(ctl_table * ctl, int write, struct file *filp,
+static int arlan_sysctl_infotxRing(ctl_table * ctl, int write,
                             void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int i;
@@ -698,11 +698,11 @@ static int arlan_sysctl_infotxRing(ctl_table * ctl, int write, struct file *filp
         SARLBNpln(u_char, txBuffer, 0x800);
 final:
         *lenp = pos;
-        retv = proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        retv = proc_dostring(ctl, write, buffer, lenp, ppos);
         return retv;
 }
 
-static int arlan_sysctl_inforxRing(ctl_table * ctl, int write, struct file *filp,
+static int arlan_sysctl_inforxRing(ctl_table * ctl, int write,
                             void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int i;
@@ -726,11 +726,11 @@ static int arlan_sysctl_inforxRing(ctl_table * ctl, int write, struct file *filp
         SARLBNpln(u_char, rxBuffer, 0x800);
 final:
         *lenp = pos;
-        retv = proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        retv = proc_dostring(ctl, write, buffer, lenp, ppos);
         return retv;
 }
 
-static int arlan_sysctl_info18(ctl_table * ctl, int write, struct file *filp,
+static int arlan_sysctl_info18(ctl_table * ctl, int write,
                         void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int i;
@@ -756,7 +756,7 @@ static int arlan_sysctl_info18(ctl_table * ctl, int write, struct file *filp,
 
 final:
         *lenp = pos;
-        retv = proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        retv = proc_dostring(ctl, write, buffer, lenp, ppos);
         return retv;
 }
 
@@ -766,7 +766,7 @@ final:
 
 static char conf_reset_result[200];
 
-static int arlan_configure(ctl_table * ctl, int write, struct file *filp,
+static int arlan_configure(ctl_table * ctl, int write,
                     void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int pos = 0;
@@ -788,10 +788,10 @@ static int arlan_configure(ctl_table * ctl, int write, struct file *filp,
                 return -1;
 
         *lenp = pos;
-        return proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        return proc_dostring(ctl, write, buffer, lenp, ppos);
 }
 
-static int arlan_sysctl_reset(ctl_table * ctl, int write, struct file *filp,
+static int arlan_sysctl_reset(ctl_table * ctl, int write,
                        void __user *buffer, size_t * lenp, loff_t *ppos)
 {
         int pos = 0;
@@ -811,7 +811,7 @@ static int arlan_sysctl_reset(ctl_table * ctl, int write, struct file *filp,
         } else
                 return -1;
         *lenp = pos + 3;
-        return proc_dostring(ctl, write, filp, buffer, lenp, ppos);
+        return proc_dostring(ctl, write, buffer, lenp, ppos);
 }
 
 
diff --git a/drivers/parport/procfs.c b/drivers/parport/procfs.c
index 554e11f9e1ce..8eefe56f1cbe 100644
--- a/drivers/parport/procfs.c
+++ b/drivers/parport/procfs.c
@@ -31,7 +31,7 @@
 #define PARPORT_MIN_SPINTIME_VALUE 1
 #define PARPORT_MAX_SPINTIME_VALUE 1000
 
-static int do_active_device(ctl_table *table, int write, struct file *filp,
+static int do_active_device(ctl_table *table, int write,
                       void __user *result, size_t *lenp, loff_t *ppos)
 {
         struct parport *port = (struct parport *)table->extra1;
@@ -68,7 +68,7 @@ static int do_active_device(ctl_table *table, int write, struct file *filp,
 }
 
 #ifdef CONFIG_PARPORT_1284
-static int do_autoprobe(ctl_table *table, int write, struct file *filp,
+static int do_autoprobe(ctl_table *table, int write,
                         void __user *result, size_t *lenp, loff_t *ppos)
 {
         struct parport_device_info *info = table->extra2;
@@ -111,7 +111,7 @@ static int do_autoprobe(ctl_table *table, int write, struct file *filp,
 #endif /* IEEE1284.3 support. */
 
 static int do_hardware_base_addr (ctl_table *table, int write,
-                                  struct file *filp, void __user *result,
+                                  void __user *result,
                                   size_t *lenp, loff_t *ppos)
 {
         struct parport *port = (struct parport *)table->extra1;
@@ -139,7 +139,7 @@ static int do_hardware_base_addr (ctl_table *table, int write,
 }
 
 static int do_hardware_irq (ctl_table *table, int write,
-                            struct file *filp, void __user *result,
+                            void __user *result,
                             size_t *lenp, loff_t *ppos)
 {
         struct parport *port = (struct parport *)table->extra1;
@@ -167,7 +167,7 @@ static int do_hardware_irq (ctl_table *table, int write,
 }
 
 static int do_hardware_dma (ctl_table *table, int write,
-                            struct file *filp, void __user *result,
+                            void __user *result,
                             size_t *lenp, loff_t *ppos)
 {
         struct parport *port = (struct parport *)table->extra1;
@@ -195,7 +195,7 @@ static int do_hardware_dma (ctl_table *table, int write,
 }
 
 static int do_hardware_modes (ctl_table *table, int write,
-                              struct file *filp, void __user *result,
+                              void __user *result,
                               size_t *lenp, loff_t *ppos)
 {
         struct parport *port = (struct parport *)table->extra1;
diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
index 36faa9a8e18f..3070f77eb56a 100644
--- a/drivers/pci/hotplug/pciehp.h
+++ b/drivers/pci/hotplug/pciehp.h
@@ -72,15 +72,9 @@ do {									\
 
 #define SLOT_NAME_SIZE 10
 struct slot {
-        u8 bus;
-        u8 device;
         u8 state;
-        u8 hp_slot;
-        u32 number;
         struct controller *ctrl;
-        struct hpc_ops *hpc_ops;
         struct hotplug_slot *hotplug_slot;
-        struct list_head        slot_list;
         struct delayed_work work;       /* work for button event */
         struct mutex lock;
 };
@@ -92,18 +86,10 @@ struct event_info {
 };
 
 struct controller {
-        struct mutex crit_sect;         /* critical section mutex */
         struct mutex ctrl_lock;         /* controller lock */
-        int num_slots;                  /* Number of slots on ctlr */
-        int slot_num_inc;               /* 1 or -1 */
-        struct pci_dev *pci_dev;
         struct pcie_device *pcie;       /* PCI Express port service */
-        struct list_head slot_list;
-        struct hpc_ops *hpc_ops;
+        struct slot *slot;
         wait_queue_head_t queue;        /* sleep & wake process */
-        u8 slot_device_offset;
-        u32 first_slot;         /* First physical slot number */  /* PCIE only has 1 slot */
-        u8 slot_bus;            /* Bus where the slots handled by this controller sit */
         u32 slot_cap;
         u8 cap_base;
         struct timer_list poll_timer;
@@ -131,40 +117,20 @@ struct controller {
 #define POWERON_STATE                   3
 #define POWEROFF_STATE                  4
 
-/* Error messages */
-#define INTERLOCK_OPEN                  0x00000002
-#define ADD_NOT_SUPPORTED               0x00000003
-#define CARD_FUNCTIONING                0x00000005
-#define ADAPTER_NOT_SAME                0x00000006
-#define NO_ADAPTER_PRESENT              0x00000009
-#define NOT_ENOUGH_RESOURCES            0x0000000B
-#define DEVICE_TYPE_NOT_SUPPORTED       0x0000000C
-#define WRONG_BUS_FREQUENCY             0x0000000D
-#define POWER_FAILURE                   0x0000000E
-
-/* Field definitions in Slot Capabilities Register */
-#define ATTN_BUTTN_PRSN 0x00000001
-#define PWR_CTRL_PRSN   0x00000002
-#define MRL_SENS_PRSN   0x00000004
-#define ATTN_LED_PRSN   0x00000008
-#define PWR_LED_PRSN    0x00000010
-#define HP_SUPR_RM_SUP  0x00000020
-#define EMI_PRSN        0x00020000
-#define NO_CMD_CMPL_SUP 0x00040000
-
-#define ATTN_BUTTN(ctrl)        ((ctrl)->slot_cap & ATTN_BUTTN_PRSN)
-#define POWER_CTRL(ctrl)        ((ctrl)->slot_cap & PWR_CTRL_PRSN)
-#define MRL_SENS(ctrl)          ((ctrl)->slot_cap & MRL_SENS_PRSN)
-#define ATTN_LED(ctrl)          ((ctrl)->slot_cap & ATTN_LED_PRSN)
-#define PWR_LED(ctrl)           ((ctrl)->slot_cap & PWR_LED_PRSN)
-#define HP_SUPR_RM(ctrl)        ((ctrl)->slot_cap & HP_SUPR_RM_SUP)
-#define EMI(ctrl)               ((ctrl)->slot_cap & EMI_PRSN)
-#define NO_CMD_CMPL(ctrl)       ((ctrl)->slot_cap & NO_CMD_CMPL_SUP)
+#define ATTN_BUTTN(ctrl)        ((ctrl)->slot_cap & PCI_EXP_SLTCAP_ABP)
+#define POWER_CTRL(ctrl)        ((ctrl)->slot_cap & PCI_EXP_SLTCAP_PCP)
+#define MRL_SENS(ctrl)          ((ctrl)->slot_cap & PCI_EXP_SLTCAP_MRLSP)
+#define ATTN_LED(ctrl)          ((ctrl)->slot_cap & PCI_EXP_SLTCAP_AIP)
+#define PWR_LED(ctrl)           ((ctrl)->slot_cap & PCI_EXP_SLTCAP_PIP)
+#define HP_SUPR_RM(ctrl)        ((ctrl)->slot_cap & PCI_EXP_SLTCAP_HPS)
+#define EMI(ctrl)               ((ctrl)->slot_cap & PCI_EXP_SLTCAP_EIP)
+#define NO_CMD_CMPL(ctrl)       ((ctrl)->slot_cap & PCI_EXP_SLTCAP_NCCS)
+#define PSN(ctrl)               ((ctrl)->slot_cap >> 19)
 
 extern int pciehp_sysfs_enable_slot(struct slot *slot);
 extern int pciehp_sysfs_disable_slot(struct slot *slot);
 extern u8 pciehp_handle_attention_button(struct slot *p_slot);
-  extern u8 pciehp_handle_switch_change(struct slot *p_slot);
+extern u8 pciehp_handle_switch_change(struct slot *p_slot);
 extern u8 pciehp_handle_presence_change(struct slot *p_slot);
 extern u8 pciehp_handle_power_fault(struct slot *p_slot);
 extern int pciehp_configure_device(struct slot *p_slot);
@@ -175,45 +141,30 @@ int pcie_init_notification(struct controller *ctrl);
 int pciehp_enable_slot(struct slot *p_slot);
 int pciehp_disable_slot(struct slot *p_slot);
 int pcie_enable_notification(struct controller *ctrl);
+int pciehp_power_on_slot(struct slot *slot);
+int pciehp_power_off_slot(struct slot *slot);
+int pciehp_get_power_status(struct slot *slot, u8 *status);
+int pciehp_get_attention_status(struct slot *slot, u8 *status);
+
+int pciehp_set_attention_status(struct slot *slot, u8 status);
+int pciehp_get_latch_status(struct slot *slot, u8 *status);
+int pciehp_get_adapter_status(struct slot *slot, u8 *status);
+int pciehp_get_max_link_speed(struct slot *slot, enum pci_bus_speed *speed);
+int pciehp_get_max_link_width(struct slot *slot, enum pcie_link_width *val);
+int pciehp_get_cur_link_speed(struct slot *slot, enum pci_bus_speed *speed);
+int pciehp_get_cur_link_width(struct slot *slot, enum pcie_link_width *val);
+int pciehp_query_power_fault(struct slot *slot);
+void pciehp_green_led_on(struct slot *slot);
+void pciehp_green_led_off(struct slot *slot);
+void pciehp_green_led_blink(struct slot *slot);
+int pciehp_check_link_status(struct controller *ctrl);
+void pciehp_release_ctrl(struct controller *ctrl);
 
 static inline const char *slot_name(struct slot *slot)
 {
         return hotplug_slot_name(slot->hotplug_slot);
 }
 
-static inline struct slot *pciehp_find_slot(struct controller *ctrl, u8 device)
-{
-        struct slot *slot;
-
-        list_for_each_entry(slot, &ctrl->slot_list, slot_list) {
-                if (slot->device == device)
-                        return slot;
-        }
-
-        ctrl_err(ctrl, "Slot (device=0x%02x) not found\n", device);
-        return NULL;
-}
-
-struct hpc_ops {
-        int (*power_on_slot)(struct slot *slot);
-        int (*power_off_slot)(struct slot *slot);
-        int (*get_power_status)(struct slot *slot, u8 *status);
-        int (*get_attention_status)(struct slot *slot, u8 *status);
-        int (*set_attention_status)(struct slot *slot, u8 status);
-        int (*get_latch_status)(struct slot *slot, u8 *status);
-        int (*get_adapter_status)(struct slot *slot, u8 *status);
-        int (*get_max_bus_speed)(struct slot *slot, enum pci_bus_speed *speed);
-        int (*get_cur_bus_speed)(struct slot *slot, enum pci_bus_speed *speed);
-        int (*get_max_lnk_width)(struct slot *slot, enum pcie_link_width *val);
-        int (*get_cur_lnk_width)(struct slot *slot, enum pcie_link_width *val);
-        int (*query_power_fault)(struct slot *slot);
-        void (*green_led_on)(struct slot *slot);
-        void (*green_led_off)(struct slot *slot);
-        void (*green_led_blink)(struct slot *slot);
-        void (*release_ctlr)(struct controller *ctrl);
-        int (*check_lnk_status)(struct controller *ctrl);
-};
-
 #ifdef CONFIG_ACPI
 #include <acpi/acpi.h>
 #include <acpi/acpi_bus.h>
diff --git a/drivers/pci/hotplug/pciehp_acpi.c b/drivers/pci/hotplug/pciehp_acpi.c
index 7163e6a6cfae..37c8d3d0323e 100644
--- a/drivers/pci/hotplug/pciehp_acpi.c
+++ b/drivers/pci/hotplug/pciehp_acpi.c
@@ -33,6 +33,11 @@
 #define PCIEHP_DETECT_AUTO      (2)
 #define PCIEHP_DETECT_DEFAULT   PCIEHP_DETECT_AUTO
 
+struct dummy_slot {
+        u32 number;
+        struct list_head list;
+};
+
 static int slot_detection_mode;
 static char *pciehp_detect_mode;
 module_param(pciehp_detect_mode, charp, 0444);
@@ -77,7 +82,7 @@ static int __init dummy_probe(struct pcie_device *dev)
         int pos;
         u32 slot_cap;
         acpi_handle handle;
-        struct slot *slot, *tmp;
+        struct dummy_slot *slot, *tmp;
         struct pci_dev *pdev = dev->port;
         /* Note: pciehp_detect_mode != PCIEHP_DETECT_ACPI here */
         if (pciehp_get_hp_hw_control_from_firmware(pdev))
@@ -89,11 +94,11 @@ static int __init dummy_probe(struct pcie_device *dev)
         if (!slot)
                 return -ENOMEM;
         slot->number = slot_cap >> 19;
-        list_for_each_entry(tmp, &dummy_slots, slot_list) {
+        list_for_each_entry(tmp, &dummy_slots, list) {
                 if (tmp->number == slot->number)
                         dup_slot_id++;
         }
-        list_add_tail(&slot->slot_list, &dummy_slots);
+        list_add_tail(&slot->list, &dummy_slots);
         handle = DEVICE_ACPI_HANDLE(&pdev->dev);
         if (!acpi_slot_detected && acpi_pci_detect_ejectable(handle))
                 acpi_slot_detected = 1;
@@ -109,11 +114,11 @@ static struct pcie_port_service_driver __initdata dummy_driver = {
 
 static int __init select_detection_mode(void)
 {
-        struct slot *slot, *tmp;
+        struct dummy_slot *slot, *tmp;
         pcie_port_service_register(&dummy_driver);
         pcie_port_service_unregister(&dummy_driver);
-        list_for_each_entry_safe(slot, tmp, &dummy_slots, slot_list) {
-                list_del(&slot->slot_list);
+        list_for_each_entry_safe(slot, tmp, &dummy_slots, list) {
+                list_del(&slot->list);
                 kfree(slot);
         }
         if (acpi_slot_detected && dup_slot_id)
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index 2317557fdee6..bc234719b1df 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -99,65 +99,55 @@ static void release_slot(struct hotplug_slot *hotplug_slot)
         kfree(hotplug_slot);
 }
 
-static int init_slots(struct controller *ctrl)
+static int init_slot(struct controller *ctrl)
 {
-        struct slot *slot;
-        struct hotplug_slot *hotplug_slot;
-        struct hotplug_slot_info *info;
+        struct slot *slot = ctrl->slot;
+        struct hotplug_slot *hotplug = NULL;
+        struct hotplug_slot_info *info = NULL;
         char name[SLOT_NAME_SIZE];
         int retval = -ENOMEM;
 
-        list_for_each_entry(slot, &ctrl->slot_list, slot_list) {
-                hotplug_slot = kzalloc(sizeof(*hotplug_slot), GFP_KERNEL);
-                if (!hotplug_slot)
-                        goto error;
-
-                info = kzalloc(sizeof(*info), GFP_KERNEL);
-                if (!info)
-                        goto error_hpslot;
-
-                /* register this slot with the hotplug pci core */
-                hotplug_slot->info = info;
-                hotplug_slot->private = slot;
-                hotplug_slot->release = &release_slot;
-                hotplug_slot->ops = &pciehp_hotplug_slot_ops;
-                slot->hotplug_slot = hotplug_slot;
-                snprintf(name, SLOT_NAME_SIZE, "%u", slot->number);
-
-                ctrl_dbg(ctrl, "Registering domain:bus:dev=%04x:%02x:%02x "
-                         "hp_slot=%x sun=%x slot_device_offset=%x\n",
-                         pci_domain_nr(ctrl->pci_dev->subordinate),
-                         slot->bus, slot->device, slot->hp_slot, slot->number,
-                         ctrl->slot_device_offset);
-                retval = pci_hp_register(hotplug_slot,
-                                         ctrl->pci_dev->subordinate,
-                                         slot->device,
-                                         name);
-                if (retval) {
-                        ctrl_err(ctrl, "pci_hp_register failed with error %d\n",
-                                 retval);
-                        goto error_info;
-                }
-                get_power_status(hotplug_slot, &info->power_status);
-                get_attention_status(hotplug_slot, &info->attention_status);
-                get_latch_status(hotplug_slot, &info->latch_status);
-                get_adapter_status(hotplug_slot, &info->adapter_status);
+        hotplug = kzalloc(sizeof(*hotplug), GFP_KERNEL);
+        if (!hotplug)
+                goto out;
+
+        info = kzalloc(sizeof(*info), GFP_KERNEL);
+        if (!info)
+                goto out;
+
+        /* register this slot with the hotplug pci core */
+        hotplug->info = info;
+        hotplug->private = slot;
+        hotplug->release = &release_slot;
+        hotplug->ops = &pciehp_hotplug_slot_ops;
+        slot->hotplug_slot = hotplug;
+        snprintf(name, SLOT_NAME_SIZE, "%u", PSN(ctrl));
+
+        ctrl_dbg(ctrl, "Registering domain:bus:dev=%04x:%02x:00 sun=%x\n",
+                 pci_domain_nr(ctrl->pcie->port->subordinate),
+                 ctrl->pcie->port->subordinate->number, PSN(ctrl));
+        retval = pci_hp_register(hotplug,
+                                 ctrl->pcie->port->subordinate, 0, name);
+        if (retval) {
+                ctrl_err(ctrl,
+                         "pci_hp_register failed with error %d\n", retval);
+                goto out;
+        }
+        get_power_status(hotplug, &info->power_status);
+        get_attention_status(hotplug, &info->attention_status);
+        get_latch_status(hotplug, &info->latch_status);
+        get_adapter_status(hotplug, &info->adapter_status);
+out:
+        if (retval) {
+                kfree(info);
+                kfree(hotplug);
         }
-
-        return 0;
-error_info:
-        kfree(info);
-error_hpslot:
-        kfree(hotplug_slot);
-error:
         return retval;
 }
 
-static void cleanup_slots(struct controller *ctrl)
+static void cleanup_slot(struct controller *ctrl)
 {
-        struct slot *slot;
-        list_for_each_entry(slot, &ctrl->slot_list, slot_list)
-                pci_hp_deregister(slot->hotplug_slot);
+        pci_hp_deregister(ctrl->slot->hotplug_slot);
 }
 
 /*
@@ -173,7 +163,7 @@ static int set_attention_status(struct hotplug_slot *hotplug_slot, u8 status)
         hotplug_slot->info->attention_status = status;
 
         if (ATTN_LED(slot->ctrl))
-                slot->hpc_ops->set_attention_status(slot, status);
+                pciehp_set_attention_status(slot, status);
 
         return 0;
 }
@@ -208,7 +198,7 @@ static int get_power_status(struct hotplug_slot *hotplug_slot, u8 *value)
         ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n",
                   __func__, slot_name(slot));
 
-        retval = slot->hpc_ops->get_power_status(slot, value);
+        retval = pciehp_get_power_status(slot, value);
         if (retval < 0)
                 *value = hotplug_slot->info->power_status;
 
@@ -223,7 +213,7 @@ static int get_attention_status(struct hotplug_slot *hotplug_slot, u8 *value)
         ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n",
                   __func__, slot_name(slot));
 
-        retval = slot->hpc_ops->get_attention_status(slot, value);
+        retval = pciehp_get_attention_status(slot, value);
         if (retval < 0)
                 *value = hotplug_slot->info->attention_status;
 
@@ -238,7 +228,7 @@ static int get_latch_status(struct hotplug_slot *hotplug_slot, u8 *value)
         ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n",
                  __func__, slot_name(slot));
 
-        retval = slot->hpc_ops->get_latch_status(slot, value);
+        retval = pciehp_get_latch_status(slot, value);
         if (retval < 0)
                 *value = hotplug_slot->info->latch_status;
 
@@ -253,7 +243,7 @@ static int get_adapter_status(struct hotplug_slot *hotplug_slot, u8 *value)
         ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n",
                  __func__, slot_name(slot));
 
-        retval = slot->hpc_ops->get_adapter_status(slot, value);
+        retval = pciehp_get_adapter_status(slot, value);
         if (retval < 0)
                 *value = hotplug_slot->info->adapter_status;
 
@@ -269,7 +259,7 @@ static int get_max_bus_speed(struct hotplug_slot *hotplug_slot,
         ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n",
                  __func__, slot_name(slot));
 
-        retval = slot->hpc_ops->get_max_bus_speed(slot, value);
+        retval = pciehp_get_max_link_speed(slot, value);
         if (retval < 0)
                 *value = PCI_SPEED_UNKNOWN;
 
@@ -284,7 +274,7 @@ static int get_cur_bus_speed(struct hotplug_slot *hotplug_slot, enum pci_bus_spe
         ctrl_dbg(slot->ctrl, "%s: physical_slot = %s\n",
                  __func__, slot_name(slot));
 
-        retval = slot->hpc_ops->get_cur_bus_speed(slot, value);
+        retval = pciehp_get_cur_link_speed(slot, value);
         if (retval < 0)
                 *value = PCI_SPEED_UNKNOWN;
 
@@ -295,7 +285,7 @@ static int pciehp_probe(struct pcie_device *dev)
 {
         int rc;
         struct controller *ctrl;
-        struct slot *t_slot;
+        struct slot *slot;
         u8 value;
         struct pci_dev *pdev = dev->port;
 
@@ -314,7 +304,7 @@ static int pciehp_probe(struct pcie_device *dev)
         set_service_data(dev, ctrl);
 
         /* Setup the slot information structures */
-        rc = init_slots(ctrl);
+        rc = init_slot(ctrl);
         if (rc) {
                 if (rc == -EBUSY)
                         ctrl_warn(ctrl, "Slot already registered by another "
@@ -332,15 +322,15 @@ static int pciehp_probe(struct pcie_device *dev)
         }
 
         /* Check if slot is occupied */
-        t_slot = pciehp_find_slot(ctrl, ctrl->slot_device_offset);
-        t_slot->hpc_ops->get_adapter_status(t_slot, &value);
+        slot = ctrl->slot;
+        pciehp_get_adapter_status(slot, &value);
         if (value) {
                 if (pciehp_force)
-                        pciehp_enable_slot(t_slot);
+                        pciehp_enable_slot(slot);
         } else {
                 /* Power off slot if not occupied */
                 if (POWER_CTRL(ctrl)) {
-                        rc = t_slot->hpc_ops->power_off_slot(t_slot);
+                        rc = pciehp_power_off_slot(slot);
                         if (rc)
                                 goto err_out_free_ctrl_slot;
                 }
@@ -349,19 +339,19 @@ static int pciehp_probe(struct pcie_device *dev)
         return 0;
 
 err_out_free_ctrl_slot:
-        cleanup_slots(ctrl);
+        cleanup_slot(ctrl);
 err_out_release_ctlr:
-        ctrl->hpc_ops->release_ctlr(ctrl);
+        pciehp_release_ctrl(ctrl);
 err_out_none:
         return -ENODEV;
 }
 
-static void pciehp_remove (struct pcie_device *dev)
+static void pciehp_remove(struct pcie_device *dev)
 {
         struct controller *ctrl = get_service_data(dev);
 
-        cleanup_slots(ctrl);
-        ctrl->hpc_ops->release_ctlr(ctrl);
+        cleanup_slot(ctrl);
+        pciehp_release_ctrl(ctrl);
 }
 
 #ifdef CONFIG_PM
@@ -376,20 +366,20 @@ static int pciehp_resume (struct pcie_device *dev)
         dev_info(&dev->device, "%s ENTRY\n", __func__);
         if (pciehp_force) {
                 struct controller *ctrl = get_service_data(dev);
-                struct slot *t_slot;
+                struct slot *slot;
                 u8 status;
 
                 /* reinitialize the chipset's event detection logic */
                 pcie_enable_notification(ctrl);
 
-                t_slot = pciehp_find_slot(ctrl, ctrl->slot_device_offset);
+                slot = ctrl->slot;
 
                 /* Check if slot is occupied */
-                t_slot->hpc_ops->get_adapter_status(t_slot, &status);
+                pciehp_get_adapter_status(slot, &status);
                 if (status)
-                        pciehp_enable_slot(t_slot);
+                        pciehp_enable_slot(slot);
                 else
-                        pciehp_disable_slot(t_slot);
+                        pciehp_disable_slot(slot);
         }
         return 0;
 }
diff --git a/drivers/pci/hotplug/pciehp_ctrl.c b/drivers/pci/hotplug/pciehp_ctrl.c
index b97cb4c3e0fe..84487d126e4d 100644
--- a/drivers/pci/hotplug/pciehp_ctrl.c
+++ b/drivers/pci/hotplug/pciehp_ctrl.c
@@ -82,7 +82,7 @@ u8 pciehp_handle_switch_change(struct slot *p_slot)
         /* Switch Change */
         ctrl_dbg(ctrl, "Switch interrupt received\n");
 
-        p_slot->hpc_ops->get_latch_status(p_slot, &getstatus);
+        pciehp_get_latch_status(p_slot, &getstatus);
         if (getstatus) {
                 /*
                  * Switch opened
@@ -114,7 +114,7 @@ u8 pciehp_handle_presence_change(struct slot *p_slot)
         /* Switch is open, assume a presence change
          * Save the presence state
          */
-        p_slot->hpc_ops->get_adapter_status(p_slot, &presence_save);
+        pciehp_get_adapter_status(p_slot, &presence_save);
         if (presence_save) {
                 /*
                  * Card Present
@@ -143,7 +143,7 @@ u8 pciehp_handle_power_fault(struct slot *p_slot)
         /* power fault */
         ctrl_dbg(ctrl, "Power fault interrupt received\n");
 
-        if ( !(p_slot->hpc_ops->query_power_fault(p_slot))) {
+        if (!pciehp_query_power_fault(p_slot)) {
                 /*
                  * power fault Cleared
                  */
@@ -172,7 +172,7 @@ static void set_slot_off(struct controller *ctrl, struct slot * pslot)
 {
         /* turn off slot, turn on Amber LED, turn off Green LED if supported*/
         if (POWER_CTRL(ctrl)) {
-                if (pslot->hpc_ops->power_off_slot(pslot)) {
+                if (pciehp_power_off_slot(pslot)) {
                         ctrl_err(ctrl,
                                  "Issue of Slot Power Off command failed\n");
                         return;
@@ -186,10 +186,10 @@ static void set_slot_off(struct controller *ctrl, struct slot * pslot)
         }
 
         if (PWR_LED(ctrl))
-                pslot->hpc_ops->green_led_off(pslot);
+                pciehp_green_led_off(pslot);
 
         if (ATTN_LED(ctrl)) {
-                if (pslot->hpc_ops->set_attention_status(pslot, 1)) {
+                if (pciehp_set_attention_status(pslot, 1)) {
                         ctrl_err(ctrl,
                                  "Issue of Set Attention Led command failed\n");
                         return;
@@ -208,24 +208,20 @@ static int board_added(struct slot *p_slot)
 {
         int retval = 0;
         struct controller *ctrl = p_slot->ctrl;
-        struct pci_bus *parent = ctrl->pci_dev->subordinate;
-
-        ctrl_dbg(ctrl, "%s: slot device, slot offset, hp slot = %d, %d, %d\n",
-                 __func__, p_slot->device, ctrl->slot_device_offset,
-                 p_slot->hp_slot);
+        struct pci_bus *parent = ctrl->pcie->port->subordinate;
 
         if (POWER_CTRL(ctrl)) {
                 /* Power on slot */
-                retval = p_slot->hpc_ops->power_on_slot(p_slot);
+                retval = pciehp_power_on_slot(p_slot);
                 if (retval)
                         return retval;
         }
 
         if (PWR_LED(ctrl))
-                p_slot->hpc_ops->green_led_blink(p_slot);
+                pciehp_green_led_blink(p_slot);
 
         /* Check link training status */
-        retval = p_slot->hpc_ops->check_lnk_status(ctrl);
+        retval = pciehp_check_link_status(ctrl);
         if (retval) {
                 ctrl_err(ctrl, "Failed to check link status\n");
                 set_slot_off(ctrl, p_slot);
@@ -233,21 +229,21 @@ static int board_added(struct slot *p_slot)
         }
 
         /* Check for a power fault */
-        if (p_slot->hpc_ops->query_power_fault(p_slot)) {
+        if (pciehp_query_power_fault(p_slot)) {
                 ctrl_dbg(ctrl, "Power fault detected\n");
-                retval = POWER_FAILURE;
+                retval = -EIO;
                 goto err_exit;
         }
 
         retval = pciehp_configure_device(p_slot);
         if (retval) {
-                ctrl_err(ctrl, "Cannot add device at %04x:%02x:%02x\n",
-                         pci_domain_nr(parent), p_slot->bus, p_slot->device);
+                ctrl_err(ctrl, "Cannot add device at %04x:%02x:00\n",
+                         pci_domain_nr(parent), parent->number);
                 goto err_exit;
         }
 
         if (PWR_LED(ctrl))
-                p_slot->hpc_ops->green_led_on(p_slot);
+                pciehp_green_led_on(p_slot);
 
         return 0;
 
@@ -269,11 +265,9 @@ static int remove_board(struct slot *p_slot)
         if (retval)
                 return retval;
 
-        ctrl_dbg(ctrl, "%s: hp_slot = %d\n", __func__, p_slot->hp_slot);
-
         if (POWER_CTRL(ctrl)) {
                 /* power off slot */
-                retval = p_slot->hpc_ops->power_off_slot(p_slot);
+                retval = pciehp_power_off_slot(p_slot);
                 if (retval) {
                         ctrl_err(ctrl,
                                  "Issue of Slot Disable command failed\n");
@@ -287,9 +281,9 @@ static int remove_board(struct slot *p_slot)
                 msleep(1000);
         }
 
+        /* turn off Green LED */
         if (PWR_LED(ctrl))
-                /* turn off Green LED */
-                p_slot->hpc_ops->green_led_off(p_slot);
+                pciehp_green_led_off(p_slot);
 
         return 0;
 }
@@ -317,18 +311,17 @@ static void pciehp_power_thread(struct work_struct *work)
         case POWEROFF_STATE:
                 mutex_unlock(&p_slot->lock);
                 ctrl_dbg(p_slot->ctrl,
-                         "Disabling domain:bus:device=%04x:%02x:%02x\n",
-                         pci_domain_nr(p_slot->ctrl->pci_dev->subordinate),
-                         p_slot->bus, p_slot->device);
+                         "Disabling domain:bus:device=%04x:%02x:00\n",
+                         pci_domain_nr(p_slot->ctrl->pcie->port->subordinate),
+                         p_slot->ctrl->pcie->port->subordinate->number);
                 pciehp_disable_slot(p_slot);
                 mutex_lock(&p_slot->lock);
                 p_slot->state = STATIC_STATE;
                 break;
         case POWERON_STATE:
                 mutex_unlock(&p_slot->lock);
-                if (pciehp_enable_slot(p_slot) &&
-                    PWR_LED(p_slot->ctrl))
-                        p_slot->hpc_ops->green_led_off(p_slot);
+                if (pciehp_enable_slot(p_slot) && PWR_LED(p_slot->ctrl))
+                        pciehp_green_led_off(p_slot);
                 mutex_lock(&p_slot->lock);
                 p_slot->state = STATIC_STATE;
                 break;
@@ -379,10 +372,10 @@ static int update_slot_info(struct slot *slot)
         if (!info)
                 return -ENOMEM;
 
-        slot->hpc_ops->get_power_status(slot, &(info->power_status));
-        slot->hpc_ops->get_attention_status(slot, &(info->attention_status));
-        slot->hpc_ops->get_latch_status(slot, &(info->latch_status));
-        slot->hpc_ops->get_adapter_status(slot, &(info->adapter_status));
+        pciehp_get_power_status(slot, &info->power_status);
+        pciehp_get_attention_status(slot, &info->attention_status);
+        pciehp_get_latch_status(slot, &info->latch_status);
+        pciehp_get_adapter_status(slot, &info->adapter_status);
 
         result = pci_hp_change_slot_info(slot->hotplug_slot, info);
         kfree (info);
@@ -399,7 +392,7 @@ static void handle_button_press_event(struct slot *p_slot)
 
         switch (p_slot->state) {
         case STATIC_STATE:
-                p_slot->hpc_ops->get_power_status(p_slot, &getstatus);
+                pciehp_get_power_status(p_slot, &getstatus);
                 if (getstatus) {
                         p_slot->state = BLINKINGOFF_STATE;
                         ctrl_info(ctrl,
@@ -413,9 +406,9 @@ static void handle_button_press_event(struct slot *p_slot)
                 }
                 /* blink green LED and turn off amber */
                 if (PWR_LED(ctrl))
-                        p_slot->hpc_ops->green_led_blink(p_slot);
+                        pciehp_green_led_blink(p_slot);
                 if (ATTN_LED(ctrl))
-                        p_slot->hpc_ops->set_attention_status(p_slot, 0);
+                        pciehp_set_attention_status(p_slot, 0);
 
                 schedule_delayed_work(&p_slot->work, 5*HZ);
                 break;
@@ -430,13 +423,13 @@ static void handle_button_press_event(struct slot *p_slot)
                 cancel_delayed_work(&p_slot->work);
                 if (p_slot->state == BLINKINGOFF_STATE) {
                         if (PWR_LED(ctrl))
-                                p_slot->hpc_ops->green_led_on(p_slot);
+                                pciehp_green_led_on(p_slot);
                 } else {
                         if (PWR_LED(ctrl))
-                                p_slot->hpc_ops->green_led_off(p_slot);
+                                pciehp_green_led_off(p_slot);
                 }
                 if (ATTN_LED(ctrl))
-                        p_slot->hpc_ops->set_attention_status(p_slot, 0);
+                        pciehp_set_attention_status(p_slot, 0);
                 ctrl_info(ctrl, "PCI slot #%s - action canceled "
                           "due to button press\n", slot_name(p_slot));
                 p_slot->state = STATIC_STATE;
@@ -474,7 +467,7 @@ static void handle_surprise_event(struct slot *p_slot)
         info->p_slot = p_slot;
         INIT_WORK(&info->work, pciehp_power_thread);
 
-        p_slot->hpc_ops->get_adapter_status(p_slot, &getstatus);
+        pciehp_get_adapter_status(p_slot, &getstatus);
         if (!getstatus)
                 p_slot->state = POWEROFF_STATE;
         else
@@ -498,9 +491,9 @@ static void interrupt_event_handler(struct work_struct *work)
                 if (!POWER_CTRL(ctrl))
                         break;
                 if (ATTN_LED(ctrl))
-                        p_slot->hpc_ops->set_attention_status(p_slot, 1);
+                        pciehp_set_attention_status(p_slot, 1);
                 if (PWR_LED(ctrl))
-                        p_slot->hpc_ops->green_led_off(p_slot);
+                        pciehp_green_led_off(p_slot);
                 break;
         case INT_PRESENCE_ON:
         case INT_PRESENCE_OFF:
@@ -525,45 +518,38 @@ int pciehp_enable_slot(struct slot *p_slot)
         int rc;
         struct controller *ctrl = p_slot->ctrl;
 
-        /* Check to see if (latch closed, card present, power off) */
-        mutex_lock(&p_slot->ctrl->crit_sect);
-
-        rc = p_slot->hpc_ops->get_adapter_status(p_slot, &getstatus);
+        rc = pciehp_get_adapter_status(p_slot, &getstatus);
         if (rc || !getstatus) {
                 ctrl_info(ctrl, "No adapter on slot(%s)\n", slot_name(p_slot));
-                mutex_unlock(&p_slot->ctrl->crit_sect);
                 return -ENODEV;
         }
         if (MRL_SENS(p_slot->ctrl)) {
-                rc = p_slot->hpc_ops->get_latch_status(p_slot, &getstatus);
+                rc = pciehp_get_latch_status(p_slot, &getstatus);
                 if (rc || getstatus) {
                         ctrl_info(ctrl, "Latch open on slot(%s)\n",
                                   slot_name(p_slot));
-                        mutex_unlock(&p_slot->ctrl->crit_sect);
                         return -ENODEV;
                 }
         }
 
         if (POWER_CTRL(p_slot->ctrl)) {
-                rc = p_slot->hpc_ops->get_power_status(p_slot, &getstatus);
+                rc = pciehp_get_power_status(p_slot, &getstatus);
                 if (rc || getstatus) {
                         ctrl_info(ctrl, "Already enabled on slot(%s)\n",
                                   slot_name(p_slot));
-                        mutex_unlock(&p_slot->ctrl->crit_sect);
                         return -EINVAL;
                 }
         }
 
-        p_slot->hpc_ops->get_latch_status(p_slot, &getstatus);
+        pciehp_get_latch_status(p_slot, &getstatus);
 
         rc = board_added(p_slot);
         if (rc) {
-                p_slot->hpc_ops->get_latch_status(p_slot, &getstatus);
+                pciehp_get_latch_status(p_slot, &getstatus);
         }
 
         update_slot_info(p_slot);
 
-        mutex_unlock(&p_slot->ctrl->crit_sect);
         return rc;
 }
 
@@ -577,35 +563,29 @@ int pciehp_disable_slot(struct slot *p_slot)
         if (!p_slot->ctrl)
                 return 1;
 
-        /* Check to see if (latch closed, card present, power on) */
-        mutex_lock(&p_slot->ctrl->crit_sect);
-
         if (!HP_SUPR_RM(p_slot->ctrl)) {
-                ret = p_slot->hpc_ops->get_adapter_status(p_slot, &getstatus);
+                ret = pciehp_get_adapter_status(p_slot, &getstatus);
                 if (ret || !getstatus) {
                         ctrl_info(ctrl, "No adapter on slot(%s)\n",
                                   slot_name(p_slot));
-                        mutex_unlock(&p_slot->ctrl->crit_sect);
                         return -ENODEV;
                 }
         }
 
         if (MRL_SENS(p_slot->ctrl)) {
-                ret = p_slot->hpc_ops->get_latch_status(p_slot, &getstatus);
+                ret = pciehp_get_latch_status(p_slot, &getstatus);
                 if (ret || getstatus) {
                         ctrl_info(ctrl, "Latch open on slot(%s)\n",
                                   slot_name(p_slot));
-                        mutex_unlock(&p_slot->ctrl->crit_sect);
                         return -ENODEV;
                 }
         }
 
         if (POWER_CTRL(p_slot->ctrl)) {
-                ret = p_slot->hpc_ops->get_power_status(p_slot, &getstatus);
+                ret = pciehp_get_power_status(p_slot, &getstatus);
                 if (ret || !getstatus) {
                         ctrl_info(ctrl, "Already disabled on slot(%s)\n",
                                   slot_name(p_slot));
-                        mutex_unlock(&p_slot->ctrl->crit_sect);
                         return -EINVAL;
                 }
         }
@@ -613,7 +593,6 @@ int pciehp_disable_slot(struct slot *p_slot)
         ret = remove_board(p_slot);
         update_slot_info(p_slot);
 
-        mutex_unlock(&p_slot->ctrl->crit_sect);
         return ret;
 }
 
diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
index 271f917b6f2c..9ef4605c1ef6 100644
--- a/drivers/pci/hotplug/pciehp_hpc.c
+++ b/drivers/pci/hotplug/pciehp_hpc.c
@@ -44,25 +44,25 @@ static atomic_t pciehp_num_controllers = ATOMIC_INIT(0);
 
 static inline int pciehp_readw(struct controller *ctrl, int reg, u16 *value)
 {
-        struct pci_dev *dev = ctrl->pci_dev;
+        struct pci_dev *dev = ctrl->pcie->port;
         return pci_read_config_word(dev, ctrl->cap_base + reg, value);
 }
 
 static inline int pciehp_readl(struct controller *ctrl, int reg, u32 *value)
 {
-        struct pci_dev *dev = ctrl->pci_dev;
+        struct pci_dev *dev = ctrl->pcie->port;
         return pci_read_config_dword(dev, ctrl->cap_base + reg, value);
 }
 
 static inline int pciehp_writew(struct controller *ctrl, int reg, u16 value)
 {
-        struct pci_dev *dev = ctrl->pci_dev;
+        struct pci_dev *dev = ctrl->pcie->port;
         return pci_write_config_word(dev, ctrl->cap_base + reg, value);
 }
 
 static inline int pciehp_writel(struct controller *ctrl, int reg, u32 value)
 {
-        struct pci_dev *dev = ctrl->pci_dev;
+        struct pci_dev *dev = ctrl->pcie->port;
         return pci_write_config_dword(dev, ctrl->cap_base + reg, value);
 }
 
@@ -266,7 +266,7 @@ static void pcie_wait_link_active(struct controller *ctrl)
         ctrl_dbg(ctrl, "Data Link Layer Link Active not set in 1000 msec\n");
 }
 
-static int hpc_check_lnk_status(struct controller *ctrl)
+int pciehp_check_link_status(struct controller *ctrl)
 {
         u16 lnk_status;
         int retval = 0;
@@ -305,7 +305,7 @@ static int hpc_check_lnk_status(struct controller *ctrl)
         return retval;
 }
 
-static int hpc_get_attention_status(struct slot *slot, u8 *status)
+int pciehp_get_attention_status(struct slot *slot, u8 *status)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_ctrl;
@@ -344,7 +344,7 @@ static int hpc_get_attention_status(struct slot *slot, u8 *status)
         return 0;
 }
 
-static int hpc_get_power_status(struct slot *slot, u8 *status)
+int pciehp_get_power_status(struct slot *slot, u8 *status)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_ctrl;
@@ -376,7 +376,7 @@ static int hpc_get_power_status(struct slot *slot, u8 *status)
         return retval;
 }
 
-static int hpc_get_latch_status(struct slot *slot, u8 *status)
+int pciehp_get_latch_status(struct slot *slot, u8 *status)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_status;
@@ -392,7 +392,7 @@ static int hpc_get_latch_status(struct slot *slot, u8 *status)
         return 0;
 }
 
-static int hpc_get_adapter_status(struct slot *slot, u8 *status)
+int pciehp_get_adapter_status(struct slot *slot, u8 *status)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_status;
@@ -408,7 +408,7 @@ static int hpc_get_adapter_status(struct slot *slot, u8 *status)
         return 0;
 }
 
-static int hpc_query_power_fault(struct slot *slot)
+int pciehp_query_power_fault(struct slot *slot)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_status;
@@ -422,7 +422,7 @@ static int hpc_query_power_fault(struct slot *slot)
         return !!(slot_status & PCI_EXP_SLTSTA_PFD);
 }
 
-static int hpc_set_attention_status(struct slot *slot, u8 value)
+int pciehp_set_attention_status(struct slot *slot, u8 value)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_cmd;
@@ -450,7 +450,7 @@ static int hpc_set_attention_status(struct slot *slot, u8 value)
         return rc;
 }
 
-static void hpc_set_green_led_on(struct slot *slot)
+void pciehp_green_led_on(struct slot *slot)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_cmd;
@@ -463,7 +463,7 @@ static void hpc_set_green_led_on(struct slot *slot)
                  __func__, ctrl->cap_base + PCI_EXP_SLTCTL, slot_cmd);
 }
 
-static void hpc_set_green_led_off(struct slot *slot)
+void pciehp_green_led_off(struct slot *slot)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_cmd;
@@ -476,7 +476,7 @@ static void hpc_set_green_led_off(struct slot *slot)
                  __func__, ctrl->cap_base + PCI_EXP_SLTCTL, slot_cmd);
 }
 
-static void hpc_set_green_led_blink(struct slot *slot)
+void pciehp_green_led_blink(struct slot *slot)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_cmd;
@@ -489,7 +489,7 @@ static void hpc_set_green_led_blink(struct slot *slot)
                  __func__, ctrl->cap_base + PCI_EXP_SLTCTL, slot_cmd);
 }
 
-static int hpc_power_on_slot(struct slot * slot)
+int pciehp_power_on_slot(struct slot * slot)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_cmd;
@@ -497,8 +497,6 @@ static int hpc_power_on_slot(struct slot * slot)
         u16 slot_status;
         int retval = 0;
 
-        ctrl_dbg(ctrl, "%s: slot->hp_slot %x\n", __func__, slot->hp_slot);
-
         /* Clear sticky power-fault bit from previous power failures */
         retval = pciehp_readw(ctrl, PCI_EXP_SLTSTA, &slot_status);
         if (retval) {
@@ -539,7 +537,7 @@ static int hpc_power_on_slot(struct slot * slot)
 
 static inline int pcie_mask_bad_dllp(struct controller *ctrl)
 {
-        struct pci_dev *dev = ctrl->pci_dev;
+        struct pci_dev *dev = ctrl->pcie->port;
         int pos;
         u32 reg;
 
@@ -556,7 +554,7 @@ static inline int pcie_mask_bad_dllp(struct controller *ctrl)
 
 static inline void pcie_unmask_bad_dllp(struct controller *ctrl)
 {
-        struct pci_dev *dev = ctrl->pci_dev;
+        struct pci_dev *dev = ctrl->pcie->port;
         u32 reg;
         int pos;
 
@@ -570,7 +568,7 @@ static inline void pcie_unmask_bad_dllp(struct controller *ctrl)
         pci_write_config_dword(dev, pos + PCI_ERR_COR_MASK, reg);
 }
 
-static int hpc_power_off_slot(struct slot * slot)
+int pciehp_power_off_slot(struct slot * slot)
 {
         struct controller *ctrl = slot->ctrl;
         u16 slot_cmd;
@@ -578,8 +576,6 @@ static int hpc_power_off_slot(struct slot * slot)
         int retval = 0;
         int changed;
 
-        ctrl_dbg(ctrl, "%s: slot->hp_slot %x\n", __func__, slot->hp_slot);
-
         /*
          * Set Bad DLLP Mask bit in Correctable Error Mask
          * Register. This is the workaround against Bad DLLP error
@@ -614,8 +610,8 @@ static int hpc_power_off_slot(struct slot * slot)
 static irqreturn_t pcie_isr(int irq, void *dev_id)
 {
         struct controller *ctrl = (struct controller *)dev_id;
+        struct slot *slot = ctrl->slot;
         u16 detected, intr_loc;
-        struct slot *p_slot;
 
         /*
          * In order to guarantee that all interrupt events are
@@ -656,29 +652,27 @@ static irqreturn_t pcie_isr(int irq, void *dev_id)
         if (!(intr_loc & ~PCI_EXP_SLTSTA_CC))
                 return IRQ_HANDLED;
 
-        p_slot = pciehp_find_slot(ctrl, ctrl->slot_device_offset);
-
         /* Check MRL Sensor Changed */
         if (intr_loc & PCI_EXP_SLTSTA_MRLSC)
-                pciehp_handle_switch_change(p_slot);
+                pciehp_handle_switch_change(slot);
 
         /* Check Attention Button Pressed */
         if (intr_loc & PCI_EXP_SLTSTA_ABP)
-                pciehp_handle_attention_button(p_slot);
+                pciehp_handle_attention_button(slot);
 
         /* Check Presence Detect Changed */
         if (intr_loc & PCI_EXP_SLTSTA_PDC)
-                pciehp_handle_presence_change(p_slot);
+                pciehp_handle_presence_change(slot);
 
         /* Check Power Fault Detected */
         if ((intr_loc & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) {
                 ctrl->power_fault_detected = 1;
-                pciehp_handle_power_fault(p_slot);
+                pciehp_handle_power_fault(slot);
         }
         return IRQ_HANDLED;
 }
 
-static int hpc_get_max_lnk_speed(struct slot *slot, enum pci_bus_speed *value)
+int pciehp_get_max_link_speed(struct slot *slot, enum pci_bus_speed *value)
 {
         struct controller *ctrl = slot->ctrl;
         enum pcie_link_speed lnk_speed;
@@ -709,7 +703,7 @@ static int hpc_get_max_lnk_speed(struct slot *slot, enum pci_bus_speed *value)
         return retval;
 }
 
-static int hpc_get_max_lnk_width(struct slot *slot,
+int pciehp_get_max_lnk_width(struct slot *slot,
                                  enum pcie_link_width *value)
 {
         struct controller *ctrl = slot->ctrl;
@@ -759,7 +753,7 @@ static int hpc_get_max_lnk_width(struct slot *slot,
         return retval;
 }
 
-static int hpc_get_cur_lnk_speed(struct slot *slot, enum pci_bus_speed *value)
+int pciehp_get_cur_link_speed(struct slot *slot, enum pci_bus_speed *value)
 {
         struct controller *ctrl = slot->ctrl;
         enum pcie_link_speed lnk_speed = PCI_SPEED_UNKNOWN;
@@ -791,7 +785,7 @@ static int hpc_get_cur_lnk_speed(struct slot *slot, enum pci_bus_speed *value)
         return retval;
 }
 
-static int hpc_get_cur_lnk_width(struct slot *slot,
+int pciehp_get_cur_lnk_width(struct slot *slot,
                                  enum pcie_link_width *value)
 {
         struct controller *ctrl = slot->ctrl;
@@ -842,30 +836,6 @@ static int hpc_get_cur_lnk_width(struct slot *slot,
         return retval;
 }
 
-static void pcie_release_ctrl(struct controller *ctrl);
-static struct hpc_ops pciehp_hpc_ops = {
-        .power_on_slot                  = hpc_power_on_slot,
-        .power_off_slot                 = hpc_power_off_slot,
-        .set_attention_status           = hpc_set_attention_status,
-        .get_power_status               = hpc_get_power_status,
-        .get_attention_status           = hpc_get_attention_status,
-        .get_latch_status               = hpc_get_latch_status,
-        .get_adapter_status             = hpc_get_adapter_status,
-
-        .get_max_bus_speed              = hpc_get_max_lnk_speed,
-        .get_cur_bus_speed              = hpc_get_cur_lnk_speed,
-        .get_max_lnk_width              = hpc_get_max_lnk_width,
-        .get_cur_lnk_width              = hpc_get_cur_lnk_width,
-
-        .query_power_fault              = hpc_query_power_fault,
-        .green_led_on                   = hpc_set_green_led_on,
-        .green_led_off                  = hpc_set_green_led_off,
-        .green_led_blink                = hpc_set_green_led_blink,
-
-        .release_ctlr                   = pcie_release_ctrl,
-        .check_lnk_status               = hpc_check_lnk_status,
-};
-
 int pcie_enable_notification(struct controller *ctrl)
 {
         u16 cmd, mask;
@@ -930,23 +900,16 @@ static int pcie_init_slot(struct controller *ctrl)
         if (!slot)
                 return -ENOMEM;
 
-        slot->hp_slot = 0;
         slot->ctrl = ctrl;
-        slot->bus = ctrl->pci_dev->subordinate->number;
-        slot->device = ctrl->slot_device_offset + slot->hp_slot;
-        slot->hpc_ops = ctrl->hpc_ops;
-        slot->number = ctrl->first_slot;
         mutex_init(&slot->lock);
         INIT_DELAYED_WORK(&slot->work, pciehp_queue_pushbutton_work);
-        list_add(&slot->slot_list, &ctrl->slot_list);
+        ctrl->slot = slot;
         return 0;
 }
 
 static void pcie_cleanup_slot(struct controller *ctrl)
 {
-        struct slot *slot;
-        slot = list_first_entry(&ctrl->slot_list, struct slot, slot_list);
-        list_del(&slot->slot_list);
+        struct slot *slot = ctrl->slot;
         cancel_delayed_work(&slot->work);
         flush_scheduled_work();
         flush_workqueue(pciehp_wq);
@@ -957,7 +920,7 @@ static inline void dbg_ctrl(struct controller *ctrl)
 {
         int i;
         u16 reg16;
-        struct pci_dev *pdev = ctrl->pci_dev;
+        struct pci_dev *pdev = ctrl->pcie->port;
 
         if (!pciehp_debug)
                 return;
@@ -980,7 +943,7 @@ static inline void dbg_ctrl(struct controller *ctrl)
                           (unsigned long long)pci_resource_start(pdev, i));
         }
         ctrl_info(ctrl, "Slot Capabilities      : 0x%08x\n", ctrl->slot_cap);
-        ctrl_info(ctrl, "  Physical Slot Number : %d\n", ctrl->first_slot);
+        ctrl_info(ctrl, "  Physical Slot Number : %d\n", PSN(ctrl));
         ctrl_info(ctrl, "  Attention Button     : %3s\n",
                   ATTN_BUTTN(ctrl) ? "yes" : "no");
         ctrl_info(ctrl, "  Power Controller     : %3s\n",
@@ -1014,10 +977,7 @@ struct controller *pcie_init(struct pcie_device *dev)
                 dev_err(&dev->device, "%s: Out of memory\n", __func__);
                 goto abort;
         }
-        INIT_LIST_HEAD(&ctrl->slot_list);
-
         ctrl->pcie = dev;
-        ctrl->pci_dev = pdev;
         ctrl->cap_base = pci_find_capability(pdev, PCI_CAP_ID_EXP);
         if (!ctrl->cap_base) {
                 ctrl_err(ctrl, "Cannot find PCI Express capability\n");
@@ -1029,11 +989,6 @@ struct controller *pcie_init(struct pcie_device *dev)
         }
 
         ctrl->slot_cap = slot_cap;
-        ctrl->first_slot = slot_cap >> 19;
-        ctrl->slot_device_offset = 0;
-        ctrl->num_slots = 1;
-        ctrl->hpc_ops = &pciehp_hpc_ops;
-        mutex_init(&ctrl->crit_sect);
         mutex_init(&ctrl->ctrl_lock);
         init_waitqueue_head(&ctrl->queue);
         dbg_ctrl(ctrl);
@@ -1089,7 +1044,7 @@ abort:
         return NULL;
 }
 
-void pcie_release_ctrl(struct controller *ctrl)
+void pciehp_release_ctrl(struct controller *ctrl)
 {
         pcie_shutdown_notification(ctrl);
         pcie_cleanup_slot(ctrl);
diff --git a/drivers/pci/hotplug/pciehp_pci.c b/drivers/pci/hotplug/pciehp_pci.c
index 02e24d63b3ee..21733108adde 100644
--- a/drivers/pci/hotplug/pciehp_pci.c
+++ b/drivers/pci/hotplug/pciehp_pci.c
@@ -63,27 +63,27 @@ static int __ref pciehp_add_bridge(struct pci_dev *dev)
 int pciehp_configure_device(struct slot *p_slot)
 {
         struct pci_dev *dev;
-        struct pci_bus *parent = p_slot->ctrl->pci_dev->subordinate;
+        struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
         int num, fn;
         struct controller *ctrl = p_slot->ctrl;
 
-        dev = pci_get_slot(parent, PCI_DEVFN(p_slot->device, 0));
+        dev = pci_get_slot(parent, PCI_DEVFN(0, 0));
         if (dev) {
                 ctrl_err(ctrl, "Device %s already exists "
-                         "at %04x:%02x:%02x, cannot hot-add\n", pci_name(dev),
-                         pci_domain_nr(parent), p_slot->bus, p_slot->device);
+                         "at %04x:%02x:00, cannot hot-add\n", pci_name(dev),
+                         pci_domain_nr(parent), parent->number);
                 pci_dev_put(dev);
                 return -EINVAL;
         }
 
-        num = pci_scan_slot(parent, PCI_DEVFN(p_slot->device, 0));
+        num = pci_scan_slot(parent, PCI_DEVFN(0, 0));
         if (num == 0) {
                 ctrl_err(ctrl, "No new device found\n");
                 return -ENODEV;
         }
 
         for (fn = 0; fn < 8; fn++) {
-                dev = pci_get_slot(parent, PCI_DEVFN(p_slot->device, fn));
+                dev = pci_get_slot(parent, PCI_DEVFN(0, fn));
                 if (!dev)
                         continue;
                 if ((dev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
@@ -111,19 +111,18 @@ int pciehp_unconfigure_device(struct slot *p_slot)
         int j;
         u8 bctl = 0;
         u8 presence = 0;
-        struct pci_bus *parent = p_slot->ctrl->pci_dev->subordinate;
+        struct pci_bus *parent = p_slot->ctrl->pcie->port->subordinate;
         u16 command;
         struct controller *ctrl = p_slot->ctrl;
 
-        ctrl_dbg(ctrl, "%s: domain:bus:dev = %04x:%02x:%02x\n",
-                 __func__, pci_domain_nr(parent), p_slot->bus, p_slot->device);
-        ret = p_slot->hpc_ops->get_adapter_status(p_slot, &presence);
+        ctrl_dbg(ctrl, "%s: domain:bus:dev = %04x:%02x:00\n",
+                 __func__, pci_domain_nr(parent), parent->number);
+        ret = pciehp_get_adapter_status(p_slot, &presence);
         if (ret)
                 presence = 0;
 
         for (j = 0; j < 8; j++) {
-                struct pci_dev* temp = pci_get_slot(parent,
-                                (p_slot->device << 3) | j);
+                struct pci_dev* temp = pci_get_slot(parent, PCI_DEVFN(0, j));
                 if (!temp)
                         continue;
                 if ((temp->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index 10c0e62bd5a8..2ce8f9ccc66e 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -318,6 +318,8 @@ static int __init aer_service_init(void)
 {
         if (pcie_aer_disable)
                 return -ENXIO;
+        if (!pci_msi_enabled())
+                return -ENXIO;
         return pcie_port_service_register(&aerdriver);
 }
 
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index f289ca9bf18d..745402e8e498 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -303,9 +303,6 @@ static void pcie_get_aspm_reg(struct pci_dev *pdev,
         pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
         pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32);
         info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
-        /* 00b and 10b are defined as "Reserved". */
-        if (info->support == PCIE_LINK_STATE_L1)
-                info->support = 0;
         info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
         info->latency_encoding_l1  = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
         pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
diff --git a/drivers/pcmcia/Makefile b/drivers/pcmcia/Makefile
index 047394d98ac2..3247828aa203 100644
--- a/drivers/pcmcia/Makefile
+++ b/drivers/pcmcia/Makefile
@@ -71,6 +71,7 @@ pxa2xx-obj-$(CONFIG_MACH_ARMCORE)		+= pxa2xx_cm_x2xx_cs.o
 pxa2xx-obj-$(CONFIG_ARCH_VIPER)                 += pxa2xx_viper.o
 pxa2xx-obj-$(CONFIG_TRIZEPS_PCMCIA)             += pxa2xx_trizeps4.o
 pxa2xx-obj-$(CONFIG_MACH_PALMTX)                += pxa2xx_palmtx.o
+pxa2xx-obj-$(CONFIG_MACH_PALMTC)                += pxa2xx_palmtc.o
 pxa2xx-obj-$(CONFIG_MACH_PALMLD)                += pxa2xx_palmld.o
 pxa2xx-obj-$(CONFIG_MACH_E740)                  += pxa2xx_e740.o
 pxa2xx-obj-$(CONFIG_MACH_STARGATE2)             += pxa2xx_stargate2.o
diff --git a/drivers/pcmcia/pxa2xx_base.c b/drivers/pcmcia/pxa2xx_base.c
index c49a7269f6d1..87e22ef8eb02 100644
--- a/drivers/pcmcia/pxa2xx_base.c
+++ b/drivers/pcmcia/pxa2xx_base.c
@@ -300,25 +300,29 @@ static int pxa2xx_drv_pcmcia_remove(struct platform_device *dev)
         return soc_common_drv_pcmcia_remove(&dev->dev);
 }
 
-static int pxa2xx_drv_pcmcia_suspend(struct platform_device *dev, pm_message_t state)
+static int pxa2xx_drv_pcmcia_suspend(struct device *dev)
 {
-        return pcmcia_socket_dev_suspend(&dev->dev, state);
+        return pcmcia_socket_dev_suspend(dev, PMSG_SUSPEND);
 }
 
-static int pxa2xx_drv_pcmcia_resume(struct platform_device *dev)
+static int pxa2xx_drv_pcmcia_resume(struct device *dev)
 {
-        pxa2xx_configure_sockets(&dev->dev);
-        return pcmcia_socket_dev_resume(&dev->dev);
+        pxa2xx_configure_sockets(dev);
+        return pcmcia_socket_dev_resume(dev);
 }
 
+static struct dev_pm_ops  pxa2xx_drv_pcmcia_pm_ops = {
+        .suspend        = pxa2xx_drv_pcmcia_suspend,
+        .resume         = pxa2xx_drv_pcmcia_resume,
+};
+
 static struct platform_driver pxa2xx_pcmcia_driver = {
         .probe          = pxa2xx_drv_pcmcia_probe,
         .remove         = pxa2xx_drv_pcmcia_remove,
-        .suspend        = pxa2xx_drv_pcmcia_suspend,
-        .resume         = pxa2xx_drv_pcmcia_resume,
         .driver         = {
                 .name   = "pxa2xx-pcmcia",
                 .owner  = THIS_MODULE,
+                .pm     = &pxa2xx_drv_pcmcia_pm_ops,
         },
 };
 
diff --git a/drivers/pcmcia/pxa2xx_palmtc.c b/drivers/pcmcia/pxa2xx_palmtc.c
new file mode 100644
index 000000000000..3a8993ed5621
--- /dev/null
+++ b/drivers/pcmcia/pxa2xx_palmtc.c
@@ -0,0 +1,230 @@
+/*
+ * linux/drivers/pcmcia/pxa2xx_palmtc.c
+ *
+ * Driver for Palm Tungsten|C PCMCIA
+ *
+ * Copyright (C) 2008 Alex Osborne <ato@meshy.org>
+ * Copyright (C) 2009 Marek Vasut <marek.vasut@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/gpio.h>
+#include <linux/delay.h>
+
+#include <asm/mach-types.h>
+#include <mach/palmtc.h>
+#include "soc_common.h"
+
+static int palmtc_pcmcia_hw_init(struct soc_pcmcia_socket *skt)
+{
+        int ret;
+
+        ret = gpio_request(GPIO_NR_PALMTC_PCMCIA_POWER1, "PCMCIA PWR1");
+        if (ret)
+                goto err1;
+        ret = gpio_direction_output(GPIO_NR_PALMTC_PCMCIA_POWER1, 0);
+        if (ret)
+                goto err2;
+
+        ret = gpio_request(GPIO_NR_PALMTC_PCMCIA_POWER2, "PCMCIA PWR2");
+        if (ret)
+                goto err2;
+        ret = gpio_direction_output(GPIO_NR_PALMTC_PCMCIA_POWER2, 0);
+        if (ret)
+                goto err3;
+
+        ret = gpio_request(GPIO_NR_PALMTC_PCMCIA_POWER3, "PCMCIA PWR3");
+        if (ret)
+                goto err3;
+        ret = gpio_direction_output(GPIO_NR_PALMTC_PCMCIA_POWER3, 0);
+        if (ret)
+                goto err4;
+
+        ret = gpio_request(GPIO_NR_PALMTC_PCMCIA_RESET, "PCMCIA RST");
+        if (ret)
+                goto err4;
+        ret = gpio_direction_output(GPIO_NR_PALMTC_PCMCIA_RESET, 1);
+        if (ret)
+                goto err5;
+
+        ret = gpio_request(GPIO_NR_PALMTC_PCMCIA_READY, "PCMCIA RDY");
+        if (ret)
+                goto err5;
+        ret = gpio_direction_input(GPIO_NR_PALMTC_PCMCIA_READY);
+        if (ret)
+                goto err6;
+
+        ret = gpio_request(GPIO_NR_PALMTC_PCMCIA_PWRREADY, "PCMCIA PWRRDY");
+        if (ret)
+                goto err6;
+        ret = gpio_direction_input(GPIO_NR_PALMTC_PCMCIA_PWRREADY);
+        if (ret)
+                goto err7;
+
+        skt->irq = IRQ_GPIO(GPIO_NR_PALMTC_PCMCIA_READY);
+        return 0;
+
+err7:
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_PWRREADY);
+err6:
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_READY);
+err5:
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_RESET);
+err4:
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_POWER3);
+err3:
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_POWER2);
+err2:
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_POWER1);
+err1:
+        return ret;
+}
+
+static void palmtc_pcmcia_hw_shutdown(struct soc_pcmcia_socket *skt)
+{
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_PWRREADY);
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_READY);
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_RESET);
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_POWER3);
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_POWER2);
+        gpio_free(GPIO_NR_PALMTC_PCMCIA_POWER1);
+}
+
+static void palmtc_pcmcia_socket_state(struct soc_pcmcia_socket *skt,
+                                        struct pcmcia_state *state)
+{
+        state->detect = 1; /* always inserted */
+        state->ready  = !!gpio_get_value(GPIO_NR_PALMTC_PCMCIA_READY);
+        state->bvd1   = 1;
+        state->bvd2   = 1;
+        state->wrprot = 0;
+        state->vs_3v  = 1;
+        state->vs_Xv  = 0;
+}
+
+static int palmtc_wifi_powerdown(void)
+{
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_RESET, 1);
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_POWER2, 0);
+        mdelay(40);
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_POWER1, 0);
+        return 0;
+}
+
+static int palmtc_wifi_powerup(void)
+{
+        int timeout = 50;
+
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_POWER3, 1);
+        mdelay(50);
+
+        /* Power up the card, 1.8V first, after a while 3.3V */
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_POWER1, 1);
+        mdelay(100);
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_POWER2, 1);
+
+        /* Wait till the card is ready */
+        while (!gpio_get_value(GPIO_NR_PALMTC_PCMCIA_PWRREADY) &&
+                timeout) {
+                mdelay(1);
+                timeout--;
+        }
+
+        /* Power down the WiFi in case of error */
+        if (!timeout) {
+                palmtc_wifi_powerdown();
+                return 1;
+        }
+
+        /* Reset the card */
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_RESET, 1);
+        mdelay(20);
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_RESET, 0);
+        mdelay(25);
+
+        gpio_set_value(GPIO_NR_PALMTC_PCMCIA_POWER3, 0);
+
+        return 0;
+}
+
+static int palmtc_pcmcia_configure_socket(struct soc_pcmcia_socket *skt,
+                                        const socket_state_t *state)
+{
+        int ret = 1;
+
+        if (state->Vcc == 0)
+                ret = palmtc_wifi_powerdown();
+        else if (state->Vcc == 33)
+                ret = palmtc_wifi_powerup();
+
+        return ret;
+}
+
+static void palmtc_pcmcia_socket_init(struct soc_pcmcia_socket *skt)
+{
+}
+
+static void palmtc_pcmcia_socket_suspend(struct soc_pcmcia_socket *skt)
+{
+}
+
+static struct pcmcia_low_level palmtc_pcmcia_ops = {
+        .owner                  = THIS_MODULE,
+
+        .first                  = 0,
+        .nr                     = 1,
+
+        .hw_init                = palmtc_pcmcia_hw_init,
+        .hw_shutdown            = palmtc_pcmcia_hw_shutdown,
+
+        .socket_state           = palmtc_pcmcia_socket_state,
+        .configure_socket       = palmtc_pcmcia_configure_socket,
+
+        .socket_init            = palmtc_pcmcia_socket_init,
+        .socket_suspend         = palmtc_pcmcia_socket_suspend,
+};
+
+static struct platform_device *palmtc_pcmcia_device;
+
+static int __init palmtc_pcmcia_init(void)
+{
+        int ret;
+
+        if (!machine_is_palmtc())
+                return -ENODEV;
+
+        palmtc_pcmcia_device = platform_device_alloc("pxa2xx-pcmcia", -1);
+        if (!palmtc_pcmcia_device)
+                return -ENOMEM;
+
+        ret = platform_device_add_data(palmtc_pcmcia_device, &palmtc_pcmcia_ops,
+                                        sizeof(palmtc_pcmcia_ops));
+
+        if (!ret)
+                ret = platform_device_add(palmtc_pcmcia_device);
+
+        if (ret)
+                platform_device_put(palmtc_pcmcia_device);
+
+        return ret;
+}
+
+static void __exit palmtc_pcmcia_exit(void)
+{
+        platform_device_unregister(palmtc_pcmcia_device);
+}
+
+module_init(palmtc_pcmcia_init);
+module_exit(palmtc_pcmcia_exit);
+
+MODULE_AUTHOR("Alex Osborne <ato@meshy.org>,"
+            " Marek Vasut <marek.vasut@gmail.com>");
+MODULE_DESCRIPTION("PCMCIA support for Palm Tungsten|C");
+MODULE_ALIAS("platform:pxa2xx-pcmcia");
+MODULE_LICENSE("GPL");
diff --git a/drivers/rtc/rtc-pxa.c b/drivers/rtc/rtc-pxa.c
index bb8cc05605ac..747ca194fad4 100644
--- a/drivers/rtc/rtc-pxa.c
+++ b/drivers/rtc/rtc-pxa.c
@@ -438,34 +438,37 @@ static int __exit pxa_rtc_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM
-static int pxa_rtc_suspend(struct platform_device *pdev, pm_message_t state)
+static int pxa_rtc_suspend(struct device *dev)
 {
-        struct pxa_rtc *pxa_rtc = platform_get_drvdata(pdev);
+        struct pxa_rtc *pxa_rtc = dev_get_drvdata(dev);
 
-        if (device_may_wakeup(&pdev->dev))
+        if (device_may_wakeup(dev))
                 enable_irq_wake(pxa_rtc->irq_Alrm);
         return 0;
 }
 
-static int pxa_rtc_resume(struct platform_device *pdev)
+static int pxa_rtc_resume(struct device *dev)
 {
-        struct pxa_rtc *pxa_rtc = platform_get_drvdata(pdev);
+        struct pxa_rtc *pxa_rtc = dev_get_drvdata(dev);
 
-        if (device_may_wakeup(&pdev->dev))
+        if (device_may_wakeup(dev))
                 disable_irq_wake(pxa_rtc->irq_Alrm);
         return 0;
 }
-#else
-#define pxa_rtc_suspend NULL
-#define pxa_rtc_resume  NULL
+
+static struct dev_pm_ops pxa_rtc_pm_ops = {
+        .suspend        = pxa_rtc_suspend,
+        .resume         = pxa_rtc_resume,
+};
 #endif
 
 static struct platform_driver pxa_rtc_driver = {
         .remove         = __exit_p(pxa_rtc_remove),
-        .suspend        = pxa_rtc_suspend,
-        .resume         = pxa_rtc_resume,
         .driver         = {
-                .name           = "pxa-rtc",
+                .name   = "pxa-rtc",
+#ifdef CONFIG_PM
+                .pm     = &pxa_rtc_pm_ops,
+#endif
         },
 };
 
diff --git a/drivers/rtc/rtc-sa1100.c b/drivers/rtc/rtc-sa1100.c
index 021b2928f0b9..29f98a70586e 100644
--- a/drivers/rtc/rtc-sa1100.c
+++ b/drivers/rtc/rtc-sa1100.c
@@ -393,31 +393,34 @@ static int sa1100_rtc_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM
-static int sa1100_rtc_suspend(struct platform_device *pdev, pm_message_t state)
+static int sa1100_rtc_suspend(struct device *dev)
 {
-        if (device_may_wakeup(&pdev->dev))
+        if (device_may_wakeup(dev))
                 enable_irq_wake(IRQ_RTCAlrm);
         return 0;
 }
 
-static int sa1100_rtc_resume(struct platform_device *pdev)
+static int sa1100_rtc_resume(struct device *dev)
 {
-        if (device_may_wakeup(&pdev->dev))
+        if (device_may_wakeup(dev))
                 disable_irq_wake(IRQ_RTCAlrm);
         return 0;
 }
-#else
-#define sa1100_rtc_suspend      NULL
-#define sa1100_rtc_resume       NULL
+
+static struct dev_pm_ops sa1100_rtc_pm_ops = {
+        .suspend        = sa1100_rtc_suspend,
+        .resume         = sa1100_rtc_resume,
+};
 #endif
 
 static struct platform_driver sa1100_rtc_driver = {
         .probe          = sa1100_rtc_probe,
         .remove         = sa1100_rtc_remove,
-        .suspend        = sa1100_rtc_suspend,
-        .resume         = sa1100_rtc_resume,
         .driver         = {
-                .name           = "sa1100-rtc",
+                .name   = "sa1100-rtc",
+#ifdef CONFIG_PM
+                .pm     = &sa1100_rtc_pm_ops,
+#endif
         },
 };
 
diff --git a/drivers/serial/cpm_uart/cpm_uart_core.c b/drivers/serial/cpm_uart/cpm_uart_core.c
index 8d349b23249a..300cea768d74 100644
--- a/drivers/serial/cpm_uart/cpm_uart_core.c
+++ b/drivers/serial/cpm_uart/cpm_uart_core.c
@@ -649,7 +649,7 @@ static int cpm_uart_tx_pump(struct uart_port *port)
         u8 *p;
         int count;
         struct uart_cpm_port *pinfo = (struct uart_cpm_port *)port;
-        struct circ_buf *xmit = &port->info->xmit;
+        struct circ_buf *xmit = &port->state->xmit;
 
         /* Handle xon/xoff */
         if (port->x_char) {
diff --git a/drivers/serial/crisv10.c b/drivers/serial/crisv10.c
index 7be52fe288eb..31f172397af3 100644
--- a/drivers/serial/crisv10.c
+++ b/drivers/serial/crisv10.c
@@ -18,6 +18,7 @@ static char *serial_version = "$Revision: 1.25 $";
 #include <linux/tty.h>
 #include <linux/tty_flip.h>
 #include <linux/major.h>
+#include <linux/smp_lock.h>
 #include <linux/string.h>
 #include <linux/fcntl.h>
 #include <linux/mm.h>
diff --git a/drivers/serial/pxa.c b/drivers/serial/pxa.c
index 6443b7ff274a..b8629d74f6a2 100644
--- a/drivers/serial/pxa.c
+++ b/drivers/serial/pxa.c
@@ -726,9 +726,10 @@ static struct uart_driver serial_pxa_reg = {
         .cons           = PXA_CONSOLE,
 };
 
-static int serial_pxa_suspend(struct platform_device *dev, pm_message_t state)
+#ifdef CONFIG_PM
+static int serial_pxa_suspend(struct device *dev)
 {
-        struct uart_pxa_port *sport = platform_get_drvdata(dev);
+        struct uart_pxa_port *sport = dev_get_drvdata(dev);
 
         if (sport)
                 uart_suspend_port(&serial_pxa_reg, &sport->port);
@@ -736,9 +737,9 @@ static int serial_pxa_suspend(struct platform_device *dev, pm_message_t state)
         return 0;
 }
 
-static int serial_pxa_resume(struct platform_device *dev)
+static int serial_pxa_resume(struct device *dev)
 {
-        struct uart_pxa_port *sport = platform_get_drvdata(dev);
+        struct uart_pxa_port *sport = dev_get_drvdata(dev);
 
         if (sport)
                 uart_resume_port(&serial_pxa_reg, &sport->port);
@@ -746,6 +747,12 @@ static int serial_pxa_resume(struct platform_device *dev)
         return 0;
 }
 
+static struct dev_pm_ops serial_pxa_pm_ops = {
+        .suspend        = serial_pxa_suspend,
+        .resume         = serial_pxa_resume,
+};
+#endif
+
 static int serial_pxa_probe(struct platform_device *dev)
 {
         struct uart_pxa_port *sport;
@@ -825,11 +832,12 @@ static struct platform_driver serial_pxa_driver = {
         .probe          = serial_pxa_probe,
         .remove         = serial_pxa_remove,
 
-        .suspend        = serial_pxa_suspend,
-        .resume         = serial_pxa_resume,
         .driver         = {
                 .name   = "pxa2xx-uart",
                 .owner  = THIS_MODULE,
+#ifdef CONFIG_PM
+                .pm     = &serial_pxa_pm_ops,
+#endif
         },
 };
 
diff --git a/drivers/spi/amba-pl022.c b/drivers/spi/amba-pl022.c
index c0f950a7cbec..958a3ffc8987 100644
--- a/drivers/spi/amba-pl022.c
+++ b/drivers/spi/amba-pl022.c
@@ -532,7 +532,7 @@ static void restore_state(struct pl022 *pl022)
         GEN_MASK_BITS(SSP_DATA_BITS_12, SSP_CR0_MASK_DSS, 0)    | \
         GEN_MASK_BITS(SSP_MICROWIRE_CHANNEL_FULL_DUPLEX, SSP_CR0_MASK_HALFDUP, 5) | \
         GEN_MASK_BITS(SSP_CLK_POL_IDLE_LOW, SSP_CR0_MASK_SPO, 6) | \
-        GEN_MASK_BITS(SSP_CLK_FALLING_EDGE, SSP_CR0_MASK_SPH, 7) | \
+        GEN_MASK_BITS(SSP_CLK_SECOND_EDGE, SSP_CR0_MASK_SPH, 7) | \
         GEN_MASK_BITS(NMDK_SSP_DEFAULT_CLKRATE, SSP_CR0_MASK_SCR, 8) | \
         GEN_MASK_BITS(SSP_BITS_8, SSP_CR0_MASK_CSS, 16) | \
         GEN_MASK_BITS(SSP_INTERFACE_MOTOROLA_SPI, SSP_CR0_MASK_FRF, 21) \
@@ -1247,8 +1247,8 @@ static int verify_controller_parameters(struct pl022 *pl022,
                 return -EINVAL;
         }
         if (chip_info->iface == SSP_INTERFACE_MOTOROLA_SPI) {
-                if ((chip_info->clk_phase != SSP_CLK_RISING_EDGE)
-                    && (chip_info->clk_phase != SSP_CLK_FALLING_EDGE)) {
+                if ((chip_info->clk_phase != SSP_CLK_FIRST_EDGE)
+                    && (chip_info->clk_phase != SSP_CLK_SECOND_EDGE)) {
                         dev_err(chip_info->dev,
                                 "Clock Phase is configured incorrectly\n");
                         return -EINVAL;
@@ -1485,7 +1485,7 @@ static int pl022_setup(struct spi_device *spi)
                 chip_info->data_size = SSP_DATA_BITS_12;
                 chip_info->rx_lev_trig = SSP_RX_1_OR_MORE_ELEM;
                 chip_info->tx_lev_trig = SSP_TX_1_OR_MORE_EMPTY_LOC;
-                chip_info->clk_phase = SSP_CLK_FALLING_EDGE;
+                chip_info->clk_phase = SSP_CLK_SECOND_EDGE;
                 chip_info->clk_pol = SSP_CLK_POL_IDLE_LOW;
                 chip_info->ctrl_len = SSP_BITS_8;
                 chip_info->wait_state = SSP_MWIRE_WAIT_ZERO;
diff --git a/drivers/spi/pxa2xx_spi.c b/drivers/spi/pxa2xx_spi.c
index 31dd56f0dae9..c8c2b693ffac 100644
--- a/drivers/spi/pxa2xx_spi.c
+++ b/drivers/spi/pxa2xx_spi.c
@@ -1668,10 +1668,9 @@ static void pxa2xx_spi_shutdown(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM
-
-static int pxa2xx_spi_suspend(struct platform_device *pdev, pm_message_t state)
+static int pxa2xx_spi_suspend(struct device *dev)
 {
-        struct driver_data *drv_data = platform_get_drvdata(pdev);
+        struct driver_data *drv_data = dev_get_drvdata(dev);
         struct ssp_device *ssp = drv_data->ssp;
         int status = 0;
 
@@ -1684,9 +1683,9 @@ static int pxa2xx_spi_suspend(struct platform_device *pdev, pm_message_t state)
         return 0;
 }
 
-static int pxa2xx_spi_resume(struct platform_device *pdev)
+static int pxa2xx_spi_resume(struct device *dev)
 {
-        struct driver_data *drv_data = platform_get_drvdata(pdev);
+        struct driver_data *drv_data = dev_get_drvdata(dev);
         struct ssp_device *ssp = drv_data->ssp;
         int status = 0;
 
@@ -1703,26 +1702,29 @@ static int pxa2xx_spi_resume(struct platform_device *pdev)
         /* Start the queue running */
         status = start_queue(drv_data);
         if (status != 0) {
-                dev_err(&pdev->dev, "problem starting queue (%d)\n", status);
+                dev_err(dev, "problem starting queue (%d)\n", status);
                 return status;
         }
 
         return 0;
 }
-#else
-#define pxa2xx_spi_suspend NULL
-#define pxa2xx_spi_resume NULL
-#endif /* CONFIG_PM */
+
+static struct dev_pm_ops pxa2xx_spi_pm_ops = {
+        .suspend        = pxa2xx_spi_suspend,
+        .resume         = pxa2xx_spi_resume,
+};
+#endif
 
 static struct platform_driver driver = {
         .driver = {
-                .name = "pxa2xx-spi",
-                .owner = THIS_MODULE,
+                .name   = "pxa2xx-spi",
+                .owner  = THIS_MODULE,
+#ifdef CONFIG_PM
+                .pm     = &pxa2xx_spi_pm_ops,
+#endif
         },
         .remove = pxa2xx_spi_remove,
         .shutdown = pxa2xx_spi_shutdown,
-        .suspend = pxa2xx_spi_suspend,
-        .resume = pxa2xx_spi_resume,
 };
 
 static int __init pxa2xx_spi_init(void)
diff --git a/drivers/staging/go7007/Makefile b/drivers/staging/go7007/Makefile
index d14ea84a01f6..1301caa7495d 100644
--- a/drivers/staging/go7007/Makefile
+++ b/drivers/staging/go7007/Makefile
@@ -32,8 +32,3 @@ endif
 
 EXTRA_CFLAGS += -Idrivers/media/dvb/frontends
 EXTRA_CFLAGS += -Idrivers/media/dvb/dvb-core
-
-# Ubuntu 8.04 has CONFIG_SND undefined, so include lum sound/config.h too
-ifeq ($(CONFIG_SND),)
-EXTRA_CFLAGS += -include sound/config.h
-endif
diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig
index ebd7237230e3..240750881d28 100644
--- a/drivers/usb/Kconfig
+++ b/drivers/usb/Kconfig
@@ -22,7 +22,6 @@ config USB_ARCH_HAS_HCD
         default y if PCMCIA && !M32R                    # sl811_cs
         default y if ARM                                # SL-811
         default y if SUPERH                             # r8a66597-hcd
-        default y if MICROBLAZE
         default PCI
 
 # many non-PCI SOC chips embed OHCI
diff --git a/drivers/usb/host/ohci-pxa27x.c b/drivers/usb/host/ohci-pxa27x.c
index b5294a9344de..f1c06202fdf2 100644
--- a/drivers/usb/host/ohci-pxa27x.c
+++ b/drivers/usb/host/ohci-pxa27x.c
@@ -481,38 +481,47 @@ static int ohci_hcd_pxa27x_drv_remove(struct platform_device *pdev)
         return 0;
 }
 
-#ifdef  CONFIG_PM
-static int ohci_hcd_pxa27x_drv_suspend(struct platform_device *pdev, pm_message_t state)
+#ifdef CONFIG_PM
+static int ohci_hcd_pxa27x_drv_suspend(struct device *dev)
 {
-        struct usb_hcd *hcd = platform_get_drvdata(pdev);
+        struct usb_hcd *hcd = dev_get_drvdata(dev);
         struct pxa27x_ohci *ohci = to_pxa27x_ohci(hcd);
 
         if (time_before(jiffies, ohci->ohci.next_statechange))
                 msleep(5);
         ohci->ohci.next_statechange = jiffies;
 
-        pxa27x_stop_hc(ohci, &pdev->dev);
+        pxa27x_stop_hc(ohci, dev);
         hcd->state = HC_STATE_SUSPENDED;
 
         return 0;
 }
 
-static int ohci_hcd_pxa27x_drv_resume(struct platform_device *pdev)
+static int ohci_hcd_pxa27x_drv_resume(struct device *dev)
 {
-        struct usb_hcd *hcd = platform_get_drvdata(pdev);
+        struct usb_hcd *hcd = dev_get_drvdata(dev);
         struct pxa27x_ohci *ohci = to_pxa27x_ohci(hcd);
+        struct pxaohci_platform_data *inf = dev->platform_data;
         int status;
 
         if (time_before(jiffies, ohci->ohci.next_statechange))
                 msleep(5);
         ohci->ohci.next_statechange = jiffies;
 
-        if ((status = pxa27x_start_hc(ohci, &pdev->dev)) < 0)
+        if ((status = pxa27x_start_hc(ohci, dev)) < 0)
                 return status;
 
+        /* Select Power Management Mode */
+        pxa27x_ohci_select_pmm(ohci, inf->port_mode);
+
         ohci_finish_controller_resume(hcd);
         return 0;
 }
+
+static struct dev_pm_ops ohci_hcd_pxa27x_pm_ops = {
+        .suspend        = ohci_hcd_pxa27x_drv_suspend,
+        .resume         = ohci_hcd_pxa27x_drv_resume,
+};
 #endif
 
 /* work with hotplug and coldplug */
@@ -522,13 +531,12 @@ static struct platform_driver ohci_hcd_pxa27x_driver = {
         .probe          = ohci_hcd_pxa27x_drv_probe,
         .remove         = ohci_hcd_pxa27x_drv_remove,
         .shutdown       = usb_hcd_platform_shutdown,
-#ifdef CONFIG_PM
-        .suspend        = ohci_hcd_pxa27x_drv_suspend,
-        .resume         = ohci_hcd_pxa27x_drv_resume,
-#endif
         .driver         = {
                 .name   = "pxa27x-ohci",
                 .owner  = THIS_MODULE,
+#ifdef CONFIG_PM
+                .pm     = &ohci_hcd_pxa27x_pm_ops,
+#endif
         },
 };
 
diff --git a/drivers/usb/serial/sierra.c b/drivers/usb/serial/sierra.c
index 68fa0e43b781..8c075b2416bb 100644
--- a/drivers/usb/serial/sierra.c
+++ b/drivers/usb/serial/sierra.c
@@ -912,6 +912,7 @@ static void sierra_release(struct usb_serial *serial)
         }
 }
 
+#ifdef CONFIG_PM
 static void stop_read_write_urbs(struct usb_serial *serial)
 {
         int i, j;
@@ -988,6 +989,10 @@ static int sierra_resume(struct usb_serial *serial)
 
         return ec ? -EIO : 0;
 }
+#else
+#define sierra_suspend NULL
+#define sierra_resume NULL
+#endif
 
 static struct usb_serial_driver sierra_device = {
         .driver = {
diff --git a/drivers/video/backlight/da903x_bl.c b/drivers/video/backlight/da903x_bl.c
index 93bb4340cc64..701a1081e199 100644
--- a/drivers/video/backlight/da903x_bl.c
+++ b/drivers/video/backlight/da903x_bl.c
@@ -154,34 +154,38 @@ static int da903x_backlight_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM
-static int da903x_backlight_suspend(struct platform_device *pdev,
-                                 pm_message_t state)
+static int da903x_backlight_suspend(struct device *dev)
 {
+        struct platform_device *pdev = to_platform_device(dev);
         struct backlight_device *bl = platform_get_drvdata(pdev);
         return da903x_backlight_set(bl, 0);
 }
 
-static int da903x_backlight_resume(struct platform_device *pdev)
+static int da903x_backlight_resume(struct device *dev)
 {
+        struct platform_device *pdev = to_platform_device(dev);
         struct backlight_device *bl = platform_get_drvdata(pdev);
 
         backlight_update_status(bl);
         return 0;
 }
-#else
-#define da903x_backlight_suspend        NULL
-#define da903x_backlight_resume         NULL
+
+static struct dev_pm_ops da903x_backlight_pm_ops = {
+        .suspend        = da903x_backlight_suspend,
+        .resume         = da903x_backlight_resume,
+};
 #endif
 
 static struct platform_driver da903x_backlight_driver = {
         .driver         = {
                 .name   = "da903x-backlight",
                 .owner  = THIS_MODULE,
+#ifdef CONFIG_PM
+                .pm     = &da903x_backlight_pm_ops,
+#endif
         },
         .probe          = da903x_backlight_probe,
         .remove         = da903x_backlight_remove,
-        .suspend        = da903x_backlight_suspend,
-        .resume         = da903x_backlight_resume,
 };
 
 static int __init da903x_backlight_init(void)
diff --git a/drivers/video/pxafb.c b/drivers/video/pxafb.c
index 6506117c134b..1820c4a24434 100644
--- a/drivers/video/pxafb.c
+++ b/drivers/video/pxafb.c
@@ -1638,24 +1638,26 @@ pxafb_freq_policy(struct notifier_block *nb, unsigned long val, void *data)
  * Power management hooks.  Note that we won't be called from IRQ context,
  * unlike the blank functions above, so we may sleep.
  */
-static int pxafb_suspend(struct platform_device *dev, pm_message_t state)
+static int pxafb_suspend(struct device *dev)
 {
-        struct pxafb_info *fbi = platform_get_drvdata(dev);
+        struct pxafb_info *fbi = dev_get_drvdata(dev);
 
         set_ctrlr_state(fbi, C_DISABLE_PM);
         return 0;
 }
 
-static int pxafb_resume(struct platform_device *dev)
+static int pxafb_resume(struct device *dev)
 {
-        struct pxafb_info *fbi = platform_get_drvdata(dev);
+        struct pxafb_info *fbi = dev_get_drvdata(dev);
 
         set_ctrlr_state(fbi, C_ENABLE_PM);
         return 0;
 }
-#else
-#define pxafb_suspend   NULL
-#define pxafb_resume    NULL
+
+static struct dev_pm_ops pxafb_pm_ops = {
+        .suspend        = pxafb_suspend,
+        .resume         = pxafb_resume,
+};
 #endif
 
 static int __devinit pxafb_init_video_memory(struct pxafb_info *fbi)
@@ -2081,6 +2083,9 @@ static int __devinit pxafb_probe(struct platform_device *dev)
                 goto failed;
         }
 
+        if (cpu_is_pxa3xx() && inf->acceleration_enabled)
+                fbi->fb.fix.accel = FB_ACCEL_PXA3XX;
+
         fbi->backlight_power = inf->pxafb_backlight_power;
         fbi->lcd_power = inf->pxafb_lcd_power;
 
@@ -2091,14 +2096,14 @@ static int __devinit pxafb_probe(struct platform_device *dev)
                 goto failed_fbi;
         }
 
-        r = request_mem_region(r->start, r->end - r->start + 1, dev->name);
+        r = request_mem_region(r->start, resource_size(r), dev->name);
         if (r == NULL) {
                 dev_err(&dev->dev, "failed to request I/O memory\n");
                 ret = -EBUSY;
                 goto failed_fbi;
         }
 
-        fbi->mmio_base = ioremap(r->start, r->end - r->start + 1);
+        fbi->mmio_base = ioremap(r->start, resource_size(r));
         if (fbi->mmio_base == NULL) {
                 dev_err(&dev->dev, "failed to map I/O memory\n");
                 ret = -EBUSY;
@@ -2197,7 +2202,7 @@ failed_free_dma:
 failed_free_io:
         iounmap(fbi->mmio_base);
 failed_free_res:
-        release_mem_region(r->start, r->end - r->start + 1);
+        release_mem_region(r->start, resource_size(r));
 failed_fbi:
         clk_put(fbi->clk);
         platform_set_drvdata(dev, NULL);
@@ -2237,7 +2242,7 @@ static int __devexit pxafb_remove(struct platform_device *dev)
         iounmap(fbi->mmio_base);
 
         r = platform_get_resource(dev, IORESOURCE_MEM, 0);
-        release_mem_region(r->start, r->end - r->start + 1);
+        release_mem_region(r->start, resource_size(r));
 
         clk_put(fbi->clk);
         kfree(fbi);
@@ -2248,11 +2253,12 @@ static int __devexit pxafb_remove(struct platform_device *dev)
 static struct platform_driver pxafb_driver = {
         .probe          = pxafb_probe,
         .remove         = __devexit_p(pxafb_remove),
-        .suspend        = pxafb_suspend,
-        .resume         = pxafb_resume,
         .driver         = {
                 .owner  = THIS_MODULE,
                 .name   = "pxa2xx-fb",
+#ifdef CONFIG_PM
+                .pm     = &pxafb_pm_ops,
+#endif
         },
 };
 
diff --git a/drivers/vlynq/vlynq.c b/drivers/vlynq/vlynq.c
index ba3d71f5c7d0..9554ad5f9af7 100644
--- a/drivers/vlynq/vlynq.c
+++ b/drivers/vlynq/vlynq.c
@@ -702,7 +702,7 @@ static int vlynq_probe(struct platform_device *pdev)
         dev->mem_start = mem_res->start;
         dev->mem_end = mem_res->end;
 
-        len = regs_res->end - regs_res->start;
+        len = resource_size(regs_res);
         if (!request_mem_region(regs_res->start, len, dev_name(&dev->dev))) {
                 printk(KERN_ERR "%s: Can't request vlynq registers\n",
                        dev_name(&dev->dev));