154 files changed, 10327 insertions, 2131 deletions

diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile
index cc9277885dd..ca2d3b34dbf 100644
--- a/drivers/gpu/Makefile
+++ b/drivers/gpu/Makefile
@@ -1 +1 @@
-obj-y                   += drm/ vga/ stub/
+obj-y                   += drm/ vga/ stub/ ion/
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index 3f46772f0cb..ba23790450e 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -101,7 +101,7 @@ static int drm_add_magic(struct drm_master *master, struct drm_file *priv,
  * Searches and unlinks the entry in drm_device::magiclist with the magic
  * number hash key, while holding the drm_device::struct_mutex lock.
  */
-static int drm_remove_magic(struct drm_master *master, drm_magic_t magic)
+int drm_remove_magic(struct drm_master *master, drm_magic_t magic)
 {
         struct drm_magic_entry *pt;
         struct drm_hash_item *hash;
@@ -136,6 +136,8 @@ static int drm_remove_magic(struct drm_master *master, drm_magic_t magic)
  * If there is a magic number in drm_file::magic then use it, otherwise
  * searches an unique non-zero magic number and add it associating it with \p
  * file_priv.
+ * This ioctl needs protection by the drm_global_mutex, which protects
+ * struct drm_file::magic and struct drm_magic_entry::priv.
  */
 int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
@@ -173,6 +175,8 @@ int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv)
  * \return zero if authentication successed, or a negative number otherwise.
  *
  * Checks if \p file_priv is associated with the magic number passed in \arg.
+ * This ioctl needs protection by the drm_global_mutex, which protects
+ * struct drm_file::magic and struct drm_magic_entry::priv.
  */
 int drm_authmagic(struct drm_device *dev, void *data,
                   struct drm_file *file_priv)
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 82db1850666..2410c4078f3 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -499,6 +499,7 @@ void drm_connector_cleanup(struct drm_connector *connector)
         mutex_lock(&dev->mode_config.mutex);
         drm_mode_object_put(dev, &connector->base);
         list_del(&connector->head);
+        dev->mode_config.num_connector--;
         mutex_unlock(&dev->mode_config.mutex);
 }
 EXPORT_SYMBOL(drm_connector_cleanup);
@@ -529,6 +530,7 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
         mutex_lock(&dev->mode_config.mutex);
         drm_mode_object_put(dev, &encoder->base);
         list_del(&encoder->head);
+        dev->mode_config.num_encoder--;
         mutex_unlock(&dev->mode_config.mutex);
 }
 EXPORT_SYMBOL(drm_encoder_cleanup);
@@ -1866,6 +1868,10 @@ int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
         }
 
         if (num_clips && clips_ptr) {
+                if (num_clips < 0 || num_clips > DRM_MODE_FB_DIRTY_MAX_CLIPS) {
+                        ret = -EINVAL;
+                        goto out_err1;
+                }
                 clips = kzalloc(num_clips * sizeof(*clips), GFP_KERNEL);
                 if (!clips) {
                         ret = -ENOMEM;
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 92369655dca..f88a9b2c977 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -560,6 +560,11 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                         mode_changed = true;
                 } else if (set->fb == NULL) {
                         mode_changed = true;
+                } else if (set->fb->depth != set->crtc->fb->depth) {
+                        mode_changed = true;
+                } else if (set->fb->bits_per_pixel !=
+                           set->crtc->fb->bits_per_pixel) {
+                        mode_changed = true;
                 } else
                         fb_changed = true;
         }
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 9d8c892d07c..9d2668a5087 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -90,7 +90,6 @@ int drm_debugfs_create_files(struct drm_info_list *files, int count,
         struct drm_device *dev = minor->dev;
         struct dentry *ent;
         struct drm_info_node *tmp;
-        char name[64];
         int i, ret;
 
         for (i = 0; i < count; i++) {
@@ -108,6 +107,9 @@ int drm_debugfs_create_files(struct drm_info_list *files, int count,
                 ent = debugfs_create_file(files[i].name, S_IFREG | S_IRUGO,
                                           root, tmp, &drm_debugfs_fops);
                 if (!ent) {
+                        char name[64];
+                        strncpy(name, root->d_name.name,
+                                                min(root->d_name.len, 64U));
                         DRM_ERROR("Cannot create /sys/kernel/debug/dri/%s/%s\n",
                                   name, files[i].name);
                         kfree(tmp);
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 09292193daf..7425e5c9bd7 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -127,6 +127,23 @@ static const u8 edid_header[] = {
         0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00
 };
 
+ /*
+ * Sanity check the header of the base EDID block.  Return 8 if the header
+ * is perfect, down to 0 if it's totally wrong.
+ */
+int drm_edid_header_is_valid(const u8 *raw_edid)
+{
+        int i, score = 0;
+
+        for (i = 0; i < sizeof(edid_header); i++)
+                if (raw_edid[i] == edid_header[i])
+                        score++;
+
+        return score;
+}
+EXPORT_SYMBOL(drm_edid_header_is_valid);
+
+
 /*
  * Sanity check the EDID block (base or extension).  Return 0 if the block
  * doesn't check out, or 1 if it's valid.
@@ -139,12 +156,7 @@ drm_edid_block_valid(u8 *raw_edid)
         struct edid *edid = (struct edid *)raw_edid;
 
         if (raw_edid[0] == 0x00) {
-                int score = 0;
-
-                for (i = 0; i < sizeof(edid_header); i++)
-                        if (raw_edid[i] == edid_header[i])
-                                score++;
-
+                int score = drm_edid_header_is_valid(raw_edid);
                 if (score == 8) ;
                 else if (score >= 6) {
                         DRM_DEBUG("Fixing EDID header, your hardware may be failing\n");
@@ -185,8 +197,8 @@ drm_edid_block_valid(u8 *raw_edid)
 bad:
         if (raw_edid) {
                 printk(KERN_ERR "Raw EDID:\n");
-                print_hex_dump_bytes(KERN_ERR, DUMP_PREFIX_NONE, raw_edid, EDID_LENGTH);
-                printk(KERN_ERR "\n");
+                print_hex_dump(KERN_ERR, " \t", DUMP_PREFIX_NONE, 16, 1,
+                               raw_edid, EDID_LENGTH, false);
         }
         return 0;
 }
@@ -1439,6 +1451,8 @@ EXPORT_SYMBOL(drm_detect_monitor_audio);
 static void drm_add_display_info(struct edid *edid,
                                  struct drm_display_info *info)
 {
+        u8 *edid_ext;
+
         info->width_mm = edid->width_cm * 10;
         info->height_mm = edid->height_cm * 10;
 
@@ -1483,6 +1497,13 @@ static void drm_add_display_info(struct edid *edid,
                 info->color_formats = DRM_COLOR_FORMAT_YCRCB444;
         if (info->color_formats & DRM_EDID_FEATURE_RGB_YCRCB422)
                 info->color_formats = DRM_COLOR_FORMAT_YCRCB422;
+
+        /* Get data from CEA blocks if present */
+        edid_ext = drm_find_cea_extension(edid);
+        if (!edid_ext)
+                return;
+
+        info->cea_rev = edid_ext[1];
 }
 
 /**
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 802b61ac313..f7c6854eb4d 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -256,7 +256,6 @@ int drm_fb_helper_panic(struct notifier_block *n, unsigned long ununsed,
 {
         printk(KERN_ERR "panic occurred, switching back to text console\n");
         return drm_fb_helper_force_kernel_mode();
-        return 0;
 }
 EXPORT_SYMBOL(drm_fb_helper_panic);
 
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 2ec7d48fc4a..c42e12cc2dd 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -486,6 +486,11 @@ int drm_release(struct inode *inode, struct file *filp)
                   (long)old_encode_dev(file_priv->minor->device),
                   dev->open_count);
 
+        /* Release any auth tokens that might point to this file_priv,
+           (do that under the drm_global_mutex) */
+        if (file_priv->magic)
+                (void) drm_remove_magic(file_priv->master, file_priv->magic);
+
         /* if the master has gone away we can't do anything with the lock */
         if (file_priv->minor->master)
                 drm_master_release(dev, filp);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 4012fe42346..186d62eb063 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -129,7 +129,7 @@ drm_gem_destroy(struct drm_device *dev)
 }
 
 /**
- * Initialize an already allocate GEM object of the specified size with
+ * Initialize an already allocated GEM object of the specified size with
  * shmfs backing store.
  */
 int drm_gem_object_init(struct drm_device *dev,
@@ -151,6 +151,27 @@ int drm_gem_object_init(struct drm_device *dev,
 EXPORT_SYMBOL(drm_gem_object_init);
 
 /**
+ * Initialize an already allocated GEM object of the specified size with
+ * no GEM provided backing store. Instead the caller is responsible for
+ * backing the object and handling it.
+ */
+int drm_gem_private_object_init(struct drm_device *dev,
+                        struct drm_gem_object *obj, size_t size)
+{
+        BUG_ON((size & (PAGE_SIZE - 1)) != 0);
+
+        obj->dev = dev;
+        obj->filp = NULL;
+
+        kref_init(&obj->refcount);
+        atomic_set(&obj->handle_count, 0);
+        obj->size = size;
+
+        return 0;
+}
+EXPORT_SYMBOL(drm_gem_private_object_init);
+
+/**
  * Allocate a GEM object of the specified size with shmfs backing store
  */
 struct drm_gem_object *
@@ -211,6 +232,8 @@ drm_gem_handle_delete(struct drm_file *filp, u32 handle)
         idr_remove(&filp->object_idr, handle);
         spin_unlock(&filp->table_lock);
 
+        if (dev->driver->gem_close_object)
+                dev->driver->gem_close_object(obj, filp);
         drm_gem_object_handle_unreference_unlocked(obj);
 
         return 0;
@@ -227,7 +250,8 @@ drm_gem_handle_create(struct drm_file *file_priv,
                        struct drm_gem_object *obj,
                        u32 *handlep)
 {
-        int     ret;
+        struct drm_device *dev = obj->dev;
+        int ret;
 
         /*
          * Get the user-visible handle using idr.
@@ -248,6 +272,15 @@ again:
                 return ret;
 
         drm_gem_object_handle_reference(obj);
+
+        if (dev->driver->gem_open_object) {
+                ret = dev->driver->gem_open_object(obj, file_priv);
+                if (ret) {
+                        drm_gem_handle_delete(file_priv, *handlep);
+                        return ret;
+                }
+        }
+
         return 0;
 }
 EXPORT_SYMBOL(drm_gem_handle_create);
@@ -402,7 +435,12 @@ drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
 static int
 drm_gem_object_release_handle(int id, void *ptr, void *data)
 {
+        struct drm_file *file_priv = data;
         struct drm_gem_object *obj = ptr;
+        struct drm_device *dev = obj->dev;
+
+        if (dev->driver->gem_close_object)
+                dev->driver->gem_close_object(obj, file_priv);
 
         drm_gem_object_handle_unreference_unlocked(obj);
 
@@ -418,7 +456,7 @@ void
 drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
 {
         idr_for_each(&file_private->object_idr,
-                     &drm_gem_object_release_handle, NULL);
+                     &drm_gem_object_release_handle, file_private);
 
         idr_remove_all(&file_private->object_idr);
         idr_destroy(&file_private->object_idr);
@@ -427,7 +465,8 @@ drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
 void
 drm_gem_object_release(struct drm_gem_object *obj)
 {
-        fput(obj->filp);
+        if (obj->filp)
+            fput(obj->filp);
 }
 EXPORT_SYMBOL(drm_gem_object_release);
 
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 2022a5c966b..3830e9e478c 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -291,11 +291,14 @@ static void drm_irq_vgaarb_nokms(void *cookie, bool state)
         if (!dev->irq_enabled)
                 return;
 
-        if (state)
-                dev->driver->irq_uninstall(dev);
-        else {
-                dev->driver->irq_preinstall(dev);
-                dev->driver->irq_postinstall(dev);
+        if (state) {
+                if (dev->driver->irq_uninstall)
+                        dev->driver->irq_uninstall(dev);
+        } else {
+                if (dev->driver->irq_preinstall)
+                        dev->driver->irq_preinstall(dev);
+                if (dev->driver->irq_postinstall)
+                        dev->driver->irq_postinstall(dev);
         }
 }
 
@@ -338,7 +341,8 @@ int drm_irq_install(struct drm_device *dev)
         DRM_DEBUG("irq=%d\n", drm_dev_to_irq(dev));
 
         /* Before installing handler */
-        dev->driver->irq_preinstall(dev);
+        if (dev->driver->irq_preinstall)
+                dev->driver->irq_preinstall(dev);
 
         /* Install handler */
         if (drm_core_check_feature(dev, DRIVER_IRQ_SHARED))
@@ -363,11 +367,16 @@ int drm_irq_install(struct drm_device *dev)
                 vga_client_register(dev->pdev, (void *)dev, drm_irq_vgaarb_nokms, NULL);
 
         /* After installing handler */
-        ret = dev->driver->irq_postinstall(dev);
+        if (dev->driver->irq_postinstall)
+                ret = dev->driver->irq_postinstall(dev);
+
         if (ret < 0) {
                 mutex_lock(&dev->struct_mutex);
                 dev->irq_enabled = 0;
                 mutex_unlock(&dev->struct_mutex);
+                if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                        vga_client_register(dev->pdev, NULL, NULL, NULL);
+                free_irq(drm_dev_to_irq(dev), dev);
         }
 
         return ret;
@@ -413,7 +422,8 @@ int drm_irq_uninstall(struct drm_device *dev)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 vga_client_register(dev->pdev, NULL, NULL, NULL);
 
-        dev->driver->irq_uninstall(dev);
+        if (dev->driver->irq_uninstall)
+                dev->driver->irq_uninstall(dev);
 
         free_irq(drm_dev_to_irq(dev), dev);
 
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index c2d32f20e2f..ad74fb4dc54 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -994,9 +994,10 @@ bool drm_mode_parse_command_line_for_connector(const char *mode_option,
 {
         const char *name;
         unsigned int namelen;
-        int res_specified = 0, bpp_specified = 0, refresh_specified = 0;
+        bool res_specified = false, bpp_specified = false, refresh_specified = false;
         unsigned int xres = 0, yres = 0, bpp = 32, refresh = 0;
-        int yres_specified = 0, cvt = 0, rb = 0, interlace = 0, margins = 0;
+        bool yres_specified = false, cvt = false, rb = false;
+        bool interlace = false, margins = false, was_digit = false;
         int i;
         enum drm_connector_force force = DRM_FORCE_UNSPECIFIED;
 
@@ -1015,54 +1016,65 @@ bool drm_mode_parse_command_line_for_connector(const char *mode_option,
         for (i = namelen-1; i >= 0; i--) {
                 switch (name[i]) {
                 case '@':
-                        namelen = i;
                         if (!refresh_specified && !bpp_specified &&
-                            !yres_specified) {
+                            !yres_specified && !cvt && !rb && was_digit) {
                                 refresh = simple_strtol(&name[i+1], NULL, 10);
-                                refresh_specified = 1;
-                                if (cvt || rb)
-                                        cvt = 0;
+                                refresh_specified = true;
+                                was_digit = false;
                         } else
                                 goto done;
                         break;
                 case '-':
-                        namelen = i;
-                        if (!bpp_specified && !yres_specified) {
+                        if (!bpp_specified && !yres_specified && !cvt &&
+                            !rb && was_digit) {
                                 bpp = simple_strtol(&name[i+1], NULL, 10);
-                                bpp_specified = 1;
-                                if (cvt || rb)
-                                        cvt = 0;
+                                bpp_specified = true;
+                                was_digit = false;
                         } else
                                 goto done;
                         break;
                 case 'x':
-                        if (!yres_specified) {
+                        if (!yres_specified && was_digit) {
                                 yres = simple_strtol(&name[i+1], NULL, 10);
-                                yres_specified = 1;
+                                yres_specified = true;
+                                was_digit = false;
                         } else
                                 goto done;
                 case '0' ... '9':
+                        was_digit = true;
                         break;
                 case 'M':
-                        if (!yres_specified)
-                                cvt = 1;
+                        if (yres_specified || cvt || was_digit)
+                                goto done;
+                        cvt = true;
                         break;
                 case 'R':
-                        if (cvt)
-                                rb = 1;
+                        if (yres_specified || cvt || rb || was_digit)
+                                goto done;
+                        rb = true;
                         break;
                 case 'm':
-                        if (!cvt)
-                                margins = 1;
+                        if (cvt || yres_specified || was_digit)
+                                goto done;
+                        margins = true;
                         break;
                 case 'i':
-                        if (!cvt)
-                                interlace = 1;
+                        if (cvt || yres_specified || was_digit)
+                                goto done;
+                        interlace = true;
                         break;
                 case 'e':
+                        if (yres_specified || bpp_specified || refresh_specified ||
+                            was_digit || (force != DRM_FORCE_UNSPECIFIED))
+                                goto done;
+
                         force = DRM_FORCE_ON;
                         break;
                 case 'D':
+                        if (yres_specified || bpp_specified || refresh_specified ||
+                            was_digit || (force != DRM_FORCE_UNSPECIFIED))
+                                goto done;
+
                         if ((connector->connector_type != DRM_MODE_CONNECTOR_DVII) &&
                             (connector->connector_type != DRM_MODE_CONNECTOR_HDMIB))
                                 force = DRM_FORCE_ON;
@@ -1070,17 +1082,37 @@ bool drm_mode_parse_command_line_for_connector(const char *mode_option,
                                 force = DRM_FORCE_ON_DIGITAL;
                         break;
                 case 'd':
+                        if (yres_specified || bpp_specified || refresh_specified ||
+                            was_digit || (force != DRM_FORCE_UNSPECIFIED))
+                                goto done;
+
                         force = DRM_FORCE_OFF;
                         break;
                 default:
                         goto done;
                 }
         }
+
         if (i < 0 && yres_specified) {
-                xres = simple_strtol(name, NULL, 10);
-                res_specified = 1;
+                char *ch;
+                xres = simple_strtol(name, &ch, 10);
+                if ((ch != NULL) && (*ch == 'x'))
+                        res_specified = true;
+                else
+                        i = ch - name;
+        } else if (!yres_specified && was_digit) {
+                /* catch mode that begins with digits but has no 'x' */
+                i = 0;
         }
 done:
+        if (i >= 0) {
+                printk(KERN_WARNING
+                        "parse error at position %i in video mode '%s'\n",
+                        i, name);
+                mode->specified = false;
+                return false;
+        }
+
         if (res_specified) {
                 mode->specified = true;
                 mode->xres = xres;
@@ -1096,9 +1128,10 @@ done:
                 mode->bpp_specified = true;
                 mode->bpp = bpp;
         }
-        mode->rb = rb ? true : false;
-        mode->cvt = cvt  ? true : false;
-        mode->interlace = interlace ? true : false;
+        mode->rb = rb;
+        mode->cvt = cvt;
+        mode->interlace = interlace;
+        mode->margins = margins;
         mode->force = force;
 
         return true;
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 7223f06d8e5..2a8b6265ad3 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -123,14 +123,15 @@ static int drm_platform_set_busid(struct drm_device *dev, struct drm_master *mas
 {
         int len, ret;
 
-        master->unique_len = 10 + strlen(dev->platformdev->name);
+        master->unique_len = 13 + strlen(dev->platformdev->name);
+        master->unique_size = master->unique_len;
         master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
 
         if (master->unique == NULL)
                 return -ENOMEM;
 
         len = snprintf(master->unique, master->unique_len,
-                       "platform:%s", dev->platformdev->name);
+                        "platform:%s:%02d", dev->platformdev->name, dev->platformdev->id);
 
         if (len > master->unique_len) {
                 DRM_ERROR("Unique buffer overflowed\n");
diff --git a/drivers/gpu/drm/drm_scatter.c b/drivers/gpu/drm/drm_scatter.c
index d15e09b0ae0..7525e0311e5 100644
--- a/drivers/gpu/drm/drm_scatter.c
+++ b/drivers/gpu/drm/drm_scatter.c
@@ -83,30 +83,26 @@ int drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather * request)
         if (dev->sg)
                 return -EINVAL;
 
-        entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
         if (!entry)
                 return -ENOMEM;
 
-        memset(entry, 0, sizeof(*entry));
         pages = (request->size + PAGE_SIZE - 1) / PAGE_SIZE;
         DRM_DEBUG("size=%ld pages=%ld\n", request->size, pages);
 
         entry->pages = pages;
-        entry->pagelist = kmalloc(pages * sizeof(*entry->pagelist), GFP_KERNEL);
+        entry->pagelist = kcalloc(pages, sizeof(*entry->pagelist), GFP_KERNEL);
         if (!entry->pagelist) {
                 kfree(entry);
                 return -ENOMEM;
         }
 
-        memset(entry->pagelist, 0, pages * sizeof(*entry->pagelist));
-
-        entry->busaddr = kmalloc(pages * sizeof(*entry->busaddr), GFP_KERNEL);
+        entry->busaddr = kcalloc(pages, sizeof(*entry->busaddr), GFP_KERNEL);
         if (!entry->busaddr) {
                 kfree(entry->pagelist);
                 kfree(entry);
                 return -ENOMEM;
         }
-        memset((void *)entry->busaddr, 0, pages * sizeof(*entry->busaddr));
 
         entry->virtual = drm_vmalloc_dma(pages << PAGE_SHIFT);
         if (!entry->virtual) {
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 0a893f7400f..3c395a59da3 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -499,7 +499,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         seq_printf(m, "Interrupts received: %d\n",
                    atomic_read(&dev_priv->irq_received));
         for (i = 0; i < I915_NUM_RINGS; i++) {
-                if (IS_GEN6(dev)) {
+                if (IS_GEN6(dev) || IS_GEN7(dev)) {
                         seq_printf(m, "Graphics Interrupt mask (%s):    %08x\n",
                                    dev_priv->ring[i].name,
                                    I915_READ_IMR(&dev_priv->ring[i]));
@@ -865,7 +865,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
                            MEMSTAT_VID_SHIFT);
                 seq_printf(m, "Current P-state: %d\n",
                            (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
-        } else if (IS_GEN6(dev)) {
+        } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
                 u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
                 u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
                 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
@@ -1123,6 +1123,44 @@ static int i915_emon_status(struct seq_file *m, void *unused)
         return 0;
 }
 
+static int i915_ring_freq_table(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int ret;
+        int gpu_freq, ia_freq;
+
+        if (!(IS_GEN6(dev) || IS_GEN7(dev))) {
+                seq_printf(m, "unsupported on this chipset\n");
+                return 0;
+        }
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        seq_printf(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\n");
+
+        for (gpu_freq = dev_priv->min_delay; gpu_freq <= dev_priv->max_delay;
+             gpu_freq++) {
+                I915_WRITE(GEN6_PCODE_DATA, gpu_freq);
+                I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
+                           GEN6_PCODE_READ_MIN_FREQ_TABLE);
+                if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
+                              GEN6_PCODE_READY) == 0, 10)) {
+                        DRM_ERROR("pcode read of freq table timed out\n");
+                        continue;
+                }
+                ia_freq = I915_READ(GEN6_PCODE_DATA);
+                seq_printf(m, "%d\t\t%d\n", gpu_freq * 50, ia_freq * 100);
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
 static int i915_gfxec(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -1300,6 +1338,155 @@ static const struct file_operations i915_wedged_fops = {
         .llseek = default_llseek,
 };
 
+static int
+i915_max_freq_open(struct inode *inode,
+                   struct file *filp)
+{
+        filp->private_data = inode->i_private;
+        return 0;
+}
+
+static ssize_t
+i915_max_freq_read(struct file *filp,
+                   char __user *ubuf,
+                   size_t max,
+                   loff_t *ppos)
+{
+        struct drm_device *dev = filp->private_data;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        char buf[80];
+        int len;
+
+        len = snprintf(buf, sizeof (buf),
+                       "max freq: %d\n", dev_priv->max_delay * 50);
+
+        if (len > sizeof (buf))
+                len = sizeof (buf);
+
+        return simple_read_from_buffer(ubuf, max, ppos, buf, len);
+}
+
+static ssize_t
+i915_max_freq_write(struct file *filp,
+                  const char __user *ubuf,
+                  size_t cnt,
+                  loff_t *ppos)
+{
+        struct drm_device *dev = filp->private_data;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        char buf[20];
+        int val = 1;
+
+        if (cnt > 0) {
+                if (cnt > sizeof (buf) - 1)
+                        return -EINVAL;
+
+                if (copy_from_user(buf, ubuf, cnt))
+                        return -EFAULT;
+                buf[cnt] = 0;
+
+                val = simple_strtoul(buf, NULL, 0);
+        }
+
+        DRM_DEBUG_DRIVER("Manually setting max freq to %d\n", val);
+
+        /*
+         * Turbo will still be enabled, but won't go above the set value.
+         */
+        dev_priv->max_delay = val / 50;
+
+        gen6_set_rps(dev, val / 50);
+
+        return cnt;
+}
+
+static const struct file_operations i915_max_freq_fops = {
+        .owner = THIS_MODULE,
+        .open = i915_max_freq_open,
+        .read = i915_max_freq_read,
+        .write = i915_max_freq_write,
+        .llseek = default_llseek,
+};
+
+static int
+i915_cache_sharing_open(struct inode *inode,
+                   struct file *filp)
+{
+        filp->private_data = inode->i_private;
+        return 0;
+}
+
+static ssize_t
+i915_cache_sharing_read(struct file *filp,
+                   char __user *ubuf,
+                   size_t max,
+                   loff_t *ppos)
+{
+        struct drm_device *dev = filp->private_data;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        char buf[80];
+        u32 snpcr;
+        int len;
+
+        mutex_lock(&dev_priv->dev->struct_mutex);
+        snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
+        mutex_unlock(&dev_priv->dev->struct_mutex);
+
+        len = snprintf(buf, sizeof (buf),
+                       "%d\n", (snpcr & GEN6_MBC_SNPCR_MASK) >>
+                       GEN6_MBC_SNPCR_SHIFT);
+
+        if (len > sizeof (buf))
+                len = sizeof (buf);
+
+        return simple_read_from_buffer(ubuf, max, ppos, buf, len);
+}
+
+static ssize_t
+i915_cache_sharing_write(struct file *filp,
+                  const char __user *ubuf,
+                  size_t cnt,
+                  loff_t *ppos)
+{
+        struct drm_device *dev = filp->private_data;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        char buf[20];
+        u32 snpcr;
+        int val = 1;
+
+        if (cnt > 0) {
+                if (cnt > sizeof (buf) - 1)
+                        return -EINVAL;
+
+                if (copy_from_user(buf, ubuf, cnt))
+                        return -EFAULT;
+                buf[cnt] = 0;
+
+                val = simple_strtoul(buf, NULL, 0);
+        }
+
+        if (val < 0 || val > 3)
+                return -EINVAL;
+
+        DRM_DEBUG_DRIVER("Manually setting uncore sharing to %d\n", val);
+
+        /* Update the cache sharing policy here as well */
+        snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
+        snpcr &= ~GEN6_MBC_SNPCR_MASK;
+        snpcr |= (val << GEN6_MBC_SNPCR_SHIFT);
+        I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
+
+        return cnt;
+}
+
+static const struct file_operations i915_cache_sharing_fops = {
+        .owner = THIS_MODULE,
+        .open = i915_cache_sharing_open,
+        .read = i915_cache_sharing_read,
+        .write = i915_cache_sharing_write,
+        .llseek = default_llseek,
+};
+
 /* As the drm_debugfs_init() routines are called before dev->dev_private is
  * allocated we need to hook into the minor for release. */
 static int
@@ -1399,6 +1586,36 @@ static int i915_forcewake_create(struct dentry *root, struct drm_minor *minor)
         return drm_add_fake_info_node(minor, ent, &i915_forcewake_fops);
 }
 
+static int i915_max_freq_create(struct dentry *root, struct drm_minor *minor)
+{
+        struct drm_device *dev = minor->dev;
+        struct dentry *ent;
+
+        ent = debugfs_create_file("i915_max_freq",
+                                  S_IRUGO | S_IWUSR,
+                                  root, dev,
+                                  &i915_max_freq_fops);
+        if (IS_ERR(ent))
+                return PTR_ERR(ent);
+
+        return drm_add_fake_info_node(minor, ent, &i915_max_freq_fops);
+}
+
+static int i915_cache_sharing_create(struct dentry *root, struct drm_minor *minor)
+{
+        struct drm_device *dev = minor->dev;
+        struct dentry *ent;
+
+        ent = debugfs_create_file("i915_cache_sharing",
+                                  S_IRUGO | S_IWUSR,
+                                  root, dev,
+                                  &i915_cache_sharing_fops);
+        if (IS_ERR(ent))
+                return PTR_ERR(ent);
+
+        return drm_add_fake_info_node(minor, ent, &i915_cache_sharing_fops);
+}
+
 static struct drm_info_list i915_debugfs_list[] = {
         {"i915_capabilities", i915_capabilities, 0},
         {"i915_gem_objects", i915_gem_object_info, 0},
@@ -1430,6 +1647,7 @@ static struct drm_info_list i915_debugfs_list[] = {
         {"i915_inttoext_table", i915_inttoext_table, 0},
         {"i915_drpc_info", i915_drpc_info, 0},
         {"i915_emon_status", i915_emon_status, 0},
+        {"i915_ring_freq_table", i915_ring_freq_table, 0},
         {"i915_gfxec", i915_gfxec, 0},
         {"i915_fbc_status", i915_fbc_status, 0},
         {"i915_sr_status", i915_sr_status, 0},
@@ -1451,6 +1669,12 @@ int i915_debugfs_init(struct drm_minor *minor)
         ret = i915_forcewake_create(minor->debugfs_root, minor);
         if (ret)
                 return ret;
+        ret = i915_max_freq_create(minor->debugfs_root, minor);
+        if (ret)
+                return ret;
+        ret = i915_cache_sharing_create(minor->debugfs_root, minor);
+        if (ret)
+                return ret;
 
         return drm_debugfs_create_files(i915_debugfs_list,
                                         I915_DEBUGFS_ENTRIES,
@@ -1465,6 +1689,10 @@ void i915_debugfs_cleanup(struct drm_minor *minor)
                                  1, minor);
         drm_debugfs_remove_files((struct drm_info_list *) &i915_wedged_fops,
                                  1, minor);
+        drm_debugfs_remove_files((struct drm_info_list *) &i915_max_freq_fops,
+                                 1, minor);
+        drm_debugfs_remove_files((struct drm_info_list *) &i915_cache_sharing_fops,
+                                 1, minor);
 }
 
 #endif /* CONFIG_DEBUG_FS */
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 296fbd66f0e..c72b590f7d8 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -61,7 +61,6 @@ static void i915_write_hws_pga(struct drm_device *dev)
 static int i915_init_phys_hws(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
-        struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         /* Program Hardware Status Page */
         dev_priv->status_page_dmah =
@@ -71,10 +70,9 @@ static int i915_init_phys_hws(struct drm_device *dev)
                 DRM_ERROR("Can not allocate hardware status page\n");
                 return -ENOMEM;
         }
-        ring->status_page.page_addr =
-                (void __force __iomem *)dev_priv->status_page_dmah->vaddr;
 
-        memset_io(ring->status_page.page_addr, 0, PAGE_SIZE);
+        memset_io((void __force __iomem *)dev_priv->status_page_dmah->vaddr,
+                  0, PAGE_SIZE);
 
         i915_write_hws_pga(dev);
 
@@ -1073,6 +1071,9 @@ static void i915_setup_compression(struct drm_device *dev, int size)
         unsigned long cfb_base;
         unsigned long ll_base = 0;
 
+        /* Just in case the BIOS is doing something questionable. */
+        intel_disable_fbc(dev);
+
         compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
         if (compressed_fb)
                 compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
@@ -1099,7 +1100,6 @@ static void i915_setup_compression(struct drm_device *dev, int size)
 
         dev_priv->cfb_size = size;
 
-        intel_disable_fbc(dev);
         dev_priv->compressed_fb = compressed_fb;
         if (HAS_PCH_SPLIT(dev))
                 I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
@@ -1453,6 +1453,14 @@ unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 
         diff1 = now - dev_priv->last_time1;
 
+        /* Prevent division-by-zero if we are asking too fast.
+         * Also, we don't get interesting results if we are polling
+         * faster than once in 10ms, so just return the saved value
+         * in such cases.
+         */
+        if (diff1 <= 10)
+                return dev_priv->chipset_power;
+
         count1 = I915_READ(DMIEC);
         count2 = I915_READ(DDREC);
         count3 = I915_READ(CSIEC);
@@ -1483,6 +1491,8 @@ unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
         dev_priv->last_count1 = total_count;
         dev_priv->last_time1 = now;
 
+        dev_priv->chipset_power = ret;
+
         return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index eb91e2dd791..f07e4252b70 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -37,38 +37,70 @@
 #include <linux/console.h>
 #include "drm_crtc_helper.h"
 
-static int i915_modeset = -1;
+static int i915_modeset __read_mostly = -1;
 module_param_named(modeset, i915_modeset, int, 0400);
+MODULE_PARM_DESC(modeset,
+                "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, "
+                "1=on, -1=force vga console preference [default])");
 
-unsigned int i915_fbpercrtc = 0;
+unsigned int i915_fbpercrtc __always_unused = 0;
 module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400);
 
-int i915_panel_ignore_lid = 0;
+int i915_panel_ignore_lid __read_mostly = 0;
 module_param_named(panel_ignore_lid, i915_panel_ignore_lid, int, 0600);
+MODULE_PARM_DESC(panel_ignore_lid,
+                "Override lid status (0=autodetect [default], 1=lid open, "
+                "-1=lid closed)");
 
-unsigned int i915_powersave = 1;
+unsigned int i915_powersave __read_mostly = 1;
 module_param_named(powersave, i915_powersave, int, 0600);
+MODULE_PARM_DESC(powersave,
+                "Enable powersavings, fbc, downclocking, etc. (default: true)");
 
-unsigned int i915_semaphores = 0;
+unsigned int i915_semaphores __read_mostly = 0;
 module_param_named(semaphores, i915_semaphores, int, 0600);
+MODULE_PARM_DESC(semaphores,
+                "Use semaphores for inter-ring sync (default: false)");
 
-unsigned int i915_enable_rc6 = 0;
+unsigned int i915_enable_rc6 __read_mostly = 0;
 module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0600);
+MODULE_PARM_DESC(i915_enable_rc6,
+                "Enable power-saving render C-state 6 (default: true)");
 
-unsigned int i915_enable_fbc = 0;
+unsigned int i915_enable_fbc __read_mostly = -1;
 module_param_named(i915_enable_fbc, i915_enable_fbc, int, 0600);
+MODULE_PARM_DESC(i915_enable_fbc,
+                "Enable frame buffer compression for power savings "
+                "(default: -1 (use per-chip default))");
 
-unsigned int i915_lvds_downclock = 0;
+unsigned int i915_lvds_downclock __read_mostly = 0;
 module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400);
+MODULE_PARM_DESC(lvds_downclock,
+                "Use panel (LVDS/eDP) downclocking for power savings "
+                "(default: false)");
 
-unsigned int i915_panel_use_ssc = 1;
+unsigned int i915_panel_use_ssc __read_mostly = 1;
 module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600);
+MODULE_PARM_DESC(lvds_use_ssc,
+                "Use Spread Spectrum Clock with panels [LVDS/eDP] "
+                "(default: true)");
 
-int i915_vbt_sdvo_panel_type = -1;
+int i915_vbt_sdvo_panel_type __read_mostly = -1;
 module_param_named(vbt_sdvo_panel_type, i915_vbt_sdvo_panel_type, int, 0600);
+MODULE_PARM_DESC(vbt_sdvo_panel_type,
+                "Override selection of SDVO panel mode in the VBT "
+                "(default: auto)");
 
-static bool i915_try_reset = true;
+static bool i915_try_reset __read_mostly = true;
 module_param_named(reset, i915_try_reset, bool, 0600);
+MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
+
+bool i915_enable_hangcheck __read_mostly = true;
+module_param_named(enable_hangcheck, i915_enable_hangcheck, bool, 0644);
+MODULE_PARM_DESC(enable_hangcheck,
+                "Periodically check GPU activity for detecting hangs. "
+                "WARNING: Disabling this can cause system wide hangs. "
+                "(default: true)");
 
 static struct drm_driver driver;
 extern int intel_agp_enabled;
@@ -345,12 +377,17 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 
 void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 {
-        int loop = 500;
-        u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
-        while (fifo < 20 && loop--) {
-                udelay(10);
-                fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+        if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES ) {
+                int loop = 500;
+                u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+                while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
+                        udelay(10);
+                        fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+                }
+                WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES);
+                dev_priv->gt_fifo_count = fifo;
         }
+        dev_priv->gt_fifo_count--;
 }
 
 static int i915_drm_freeze(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index ce7914c4c04..1a2a2d1790b 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -36,6 +36,7 @@
 #include <linux/io-mapping.h>
 #include <linux/i2c.h>
 #include <drm/intel-gtt.h>
+#include <linux/backlight.h>
 
 /* General customization:
  */
@@ -214,6 +215,8 @@ struct drm_i915_display_funcs {
         int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
                           struct drm_framebuffer *fb,
                           struct drm_i915_gem_object *obj);
+        int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                            int x, int y);
         /* clock updates for mode set */
         /* cursor updates */
         /* render clock increase/decrease */
@@ -265,6 +268,7 @@ enum intel_pch {
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
 
 struct intel_fbdev;
+struct intel_fbc_work;
 
 typedef struct drm_i915_private {
         struct drm_device *dev;
@@ -275,6 +279,7 @@ typedef struct drm_i915_private {
         int relative_constants_mode;
 
         void __iomem *regs;
+        u32 gt_fifo_count;
 
         struct intel_gmbus {
                 struct i2c_adapter adapter;
@@ -329,11 +334,10 @@ typedef struct drm_i915_private {
         uint32_t last_instdone1;
 
         unsigned long cfb_size;
-        unsigned long cfb_pitch;
-        unsigned long cfb_offset;
-        int cfb_fence;
-        int cfb_plane;
+        unsigned int cfb_fb;
+        enum plane cfb_plane;
         int cfb_y;
+        struct intel_fbc_work *fbc_work;
 
         struct intel_opregion opregion;
 
@@ -541,6 +545,7 @@ typedef struct drm_i915_private {
         u32 savePIPEB_LINK_M1;
         u32 savePIPEB_LINK_N1;
         u32 saveMCHBAR_RENDER_STANDBY;
+        u32 savePCH_PORT_HOTPLUG;
 
         struct {
                 /** Bridge to intel-gtt-ko */
@@ -686,6 +691,7 @@ typedef struct drm_i915_private {
         int child_dev_num;
         struct child_device_config *child_dev;
         struct drm_connector *int_lvds_connector;
+        struct drm_connector *int_edp_connector;
 
         bool mchbar_need_disable;
 
@@ -701,6 +707,7 @@ typedef struct drm_i915_private {
 
         u64 last_count1;
         unsigned long last_time1;
+        unsigned long chipset_power;
         u64 last_count2;
         struct timespec last_time2;
         unsigned long gfx_power;
@@ -719,6 +726,8 @@ typedef struct drm_i915_private {
         /* list of fbdev register on this device */
         struct intel_fbdev *fbdev;
 
+        struct backlight_device *backlight;
+
         struct drm_property *broadcast_rgb_property;
         struct drm_property *force_audio_property;
 
@@ -986,15 +995,16 @@ struct drm_i915_file_private {
 
 extern struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
-extern unsigned int i915_fbpercrtc;
-extern int i915_panel_ignore_lid;
-extern unsigned int i915_powersave;
-extern unsigned int i915_semaphores;
-extern unsigned int i915_lvds_downclock;
-extern unsigned int i915_panel_use_ssc;
-extern int i915_vbt_sdvo_panel_type;
-extern unsigned int i915_enable_rc6;
-extern unsigned int i915_enable_fbc;
+extern unsigned int i915_fbpercrtc __always_unused;
+extern int i915_panel_ignore_lid __read_mostly;
+extern unsigned int i915_powersave __read_mostly;
+extern unsigned int i915_semaphores __read_mostly;
+extern unsigned int i915_lvds_downclock __read_mostly;
+extern unsigned int i915_panel_use_ssc __read_mostly;
+extern int i915_vbt_sdvo_panel_type __read_mostly;
+extern unsigned int i915_enable_rc6 __read_mostly;
+extern unsigned int i915_enable_fbc __read_mostly;
+extern bool i915_enable_hangcheck __read_mostly;
 
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
@@ -1164,7 +1174,7 @@ void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
                                             uint32_t read_domains,
                                             uint32_t write_domain);
-int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
 void i915_gem_do_init(struct drm_device *dev,
@@ -1183,7 +1193,8 @@ int __must_check
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
                                   bool write);
 int __must_check
-i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
+                                     u32 alignment,
                                      struct intel_ring_buffer *pipelined);
 int i915_gem_attach_phys_object(struct drm_device *dev,
                                 struct drm_i915_gem_object *obj,
@@ -1199,9 +1210,14 @@ i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
                                     uint32_t size,
                                     int tiling_mode);
 
+int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
+                                    enum i915_cache_level cache_level);
+
 /* i915_gem_gtt.c */
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
+void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
+                                enum i915_cache_level cache_level);
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 
 /* i915_gem_evict.c */
@@ -1283,12 +1299,8 @@ extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
-extern void i8xx_disable_fbc(struct drm_device *dev);
-extern void g4x_disable_fbc(struct drm_device *dev);
-extern void ironlake_disable_fbc(struct drm_device *dev);
-extern void intel_disable_fbc(struct drm_device *dev);
-extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval);
 extern bool intel_fbc_enabled(struct drm_device *dev);
+extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
 extern void ironlake_enable_rc6(struct drm_device *dev);
 extern void gen6_set_rps(struct drm_device *dev, u8 val);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index a087e1bf0c2..346d5574f0a 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1475,7 +1475,7 @@ i915_gem_mmap_gtt(struct drm_file *file,
 
         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
                 ret = -E2BIG;
-                goto unlock;
+                goto out;
         }
 
         if (obj->madv != I915_MADV_WILLNEED) {
@@ -1763,8 +1763,11 @@ i915_add_request(struct intel_ring_buffer *ring,
         ring->outstanding_lazy_request = false;
 
         if (!dev_priv->mm.suspended) {
-                mod_timer(&dev_priv->hangcheck_timer,
-                          jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+                if (i915_enable_hangcheck) {
+                        mod_timer(&dev_priv->hangcheck_timer,
+                                  jiffies +
+                                  msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+                }
                 if (was_empty)
                         queue_delayed_work(dev_priv->wq,
                                            &dev_priv->mm.retire_work, HZ);
@@ -2135,6 +2138,30 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
         return 0;
 }
 
+static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
+{
+        u32 old_write_domain, old_read_domains;
+
+        /* Act a barrier for all accesses through the GTT */
+        mb();
+
+        /* Force a pagefault for domain tracking on next user access */
+        i915_gem_release_mmap(obj);
+
+        if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
+                return;
+
+        old_read_domains = obj->base.read_domains;
+        old_write_domain = obj->base.write_domain;
+
+        obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
+        obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
+
+        trace_i915_gem_object_change_domain(obj,
+                                            old_read_domains,
+                                            old_write_domain);
+}
+
 /**
  * Unbinds an object from the GTT aperture.
  */
@@ -2151,23 +2178,28 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
                 return -EINVAL;
         }
 
-        /* blow away mappings if mapped through GTT */
-        i915_gem_release_mmap(obj);
-
-        /* Move the object to the CPU domain to ensure that
-         * any possible CPU writes while it's not in the GTT
-         * are flushed when we go to remap it. This will
-         * also ensure that all pending GPU writes are finished
-         * before we unbind.
-         */
-        ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+        ret = i915_gem_object_finish_gpu(obj);
         if (ret == -ERESTARTSYS)
                 return ret;
         /* Continue on if we fail due to EIO, the GPU is hung so we
          * should be safe and we need to cleanup or else we might
          * cause memory corruption through use-after-free.
          */
+
+        i915_gem_object_finish_gtt(obj);
+
+        /* Move the object to the CPU domain to ensure that
+         * any possible CPU writes while it's not in the GTT
+         * are flushed when we go to remap it.
+         */
+        if (ret == 0)
+                ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+        if (ret == -ERESTARTSYS)
+                return ret;
         if (ret) {
+                /* In the event of a disaster, abandon all caches and
+                 * hope for the best.
+                 */
                 i915_gem_clflush_object(obj);
                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
         }
@@ -2996,51 +3028,139 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
         return 0;
 }
 
+int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
+                                    enum i915_cache_level cache_level)
+{
+        int ret;
+
+        if (obj->cache_level == cache_level)
+                return 0;
+
+        if (obj->pin_count) {
+                DRM_DEBUG("can not change the cache level of pinned objects\n");
+                return -EBUSY;
+        }
+
+        if (obj->gtt_space) {
+                ret = i915_gem_object_finish_gpu(obj);
+                if (ret)
+                        return ret;
+
+                i915_gem_object_finish_gtt(obj);
+
+                /* Before SandyBridge, you could not use tiling or fence
+                 * registers with snooped memory, so relinquish any fences
+                 * currently pointing to our region in the aperture.
+                 */
+                if (INTEL_INFO(obj->base.dev)->gen < 6) {
+                        ret = i915_gem_object_put_fence(obj);
+                        if (ret)
+                                return ret;
+                }
+
+                i915_gem_gtt_rebind_object(obj, cache_level);
+        }
+
+        if (cache_level == I915_CACHE_NONE) {
+                u32 old_read_domains, old_write_domain;
+
+                /* If we're coming from LLC cached, then we haven't
+                 * actually been tracking whether the data is in the
+                 * CPU cache or not, since we only allow one bit set
+                 * in obj->write_domain and have been skipping the clflushes.
+                 * Just set it to the CPU cache for now.
+                 */
+                WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
+                WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
+
+                old_read_domains = obj->base.read_domains;
+                old_write_domain = obj->base.write_domain;
+
+                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+
+                trace_i915_gem_object_change_domain(obj,
+                                                    old_read_domains,
+                                                    old_write_domain);
+        }
+
+        obj->cache_level = cache_level;
+        return 0;
+}
+
 /*
- * Prepare buffer for display plane. Use uninterruptible for possible flush
- * wait, as in modesetting process we're not supposed to be interrupted.
+ * Prepare buffer for display plane (scanout, cursors, etc).
+ * Can be called from an uninterruptible phase (modesetting) and allows
+ * any flushes to be pipelined (for pageflips).
+ *
+ * For the display plane, we want to be in the GTT but out of any write
+ * domains. So in many ways this looks like set_to_gtt_domain() apart from the
+ * ability to pipeline the waits, pinning and any additional subtleties
+ * that may differentiate the display plane from ordinary buffers.
  */
 int
-i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
+                                     u32 alignment,
                                      struct intel_ring_buffer *pipelined)
 {
-        uint32_t old_read_domains;
+        u32 old_read_domains, old_write_domain;
         int ret;
 
-        /* Not valid to be called on unbound objects. */
-        if (obj->gtt_space == NULL)
-                return -EINVAL;
-
         ret = i915_gem_object_flush_gpu_write_domain(obj);
         if (ret)
                 return ret;
 
-
-        /* Currently, we are always called from an non-interruptible context. */
         if (pipelined != obj->ring) {
                 ret = i915_gem_object_wait_rendering(obj);
-                if (ret)
+                if (ret == -ERESTARTSYS)
                         return ret;
         }
 
+        /* The display engine is not coherent with the LLC cache on gen6.  As
+         * a result, we make sure that the pinning that is about to occur is
+         * done with uncached PTEs. This is lowest common denominator for all
+         * chipsets.
+         *
+         * However for gen6+, we could do better by using the GFDT bit instead
+         * of uncaching, which would allow us to flush all the LLC-cached data
+         * with that bit in the PTE to main memory with just one PIPE_CONTROL.
+         */
+        ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
+        if (ret)
+                return ret;
+
+        /* As the user may map the buffer once pinned in the display plane
+         * (e.g. libkms for the bootup splash), we have to ensure that we
+         * always use map_and_fenceable for all scanout buffers.
+         */
+        ret = i915_gem_object_pin(obj, alignment, true);
+        if (ret)
+                return ret;
+
         i915_gem_object_flush_cpu_write_domain(obj);
 
+        old_write_domain = obj->base.write_domain;
         old_read_domains = obj->base.read_domains;
+
+        /* It should now be out of any other write domains, and we can update
+         * the domain values for our changes.
+         */
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
 
         trace_i915_gem_object_change_domain(obj,
                                             old_read_domains,
-                                            obj->base.write_domain);
+                                            old_write_domain);
 
         return 0;
 }
 
 int
-i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
+i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
 {
         int ret;
 
-        if (!obj->active)
+        if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
                 return 0;
 
         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
@@ -3049,6 +3169,9 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
                         return ret;
         }
 
+        /* Ensure that we invalidate the GPU's caches and TLBs. */
+        obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
+
         return i915_gem_object_wait_rendering(obj);
 }
 
@@ -3575,7 +3698,23 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 
-        obj->cache_level = I915_CACHE_NONE;
+        if (IS_GEN6(dev)) {
+                /* On Gen6, we can have the GPU use the LLC (the CPU
+                 * cache) for about a 10% performance improvement
+                 * compared to uncached.  Graphics requests other than
+                 * display scanout are coherent with the CPU in
+                 * accessing this cache.  This means in this mode we
+                 * don't need to clflush on the CPU side, and on the
+                 * GPU side we only need to flush internal caches to
+                 * get data visible to the CPU.
+                 *
+                 * However, we maintain the display planes as UC, and so
+                 * need to rebind when first used as such.
+                 */
+                obj->cache_level = I915_CACHE_LLC;
+        } else
+                obj->cache_level = I915_CACHE_NONE;
+
         obj->base.driver_private = NULL;
         obj->fence_reg = I915_FENCE_REG_NONE;
         INIT_LIST_HEAD(&obj->mm_list);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index e46b645773c..7a709cd8d54 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -59,24 +59,8 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 
         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
-                unsigned int agp_type =
-                        cache_level_to_agp_type(dev, obj->cache_level);
-
                 i915_gem_clflush_object(obj);
-
-                if (dev_priv->mm.gtt->needs_dmar) {
-                        BUG_ON(!obj->sg_list);
-
-                        intel_gtt_insert_sg_entries(obj->sg_list,
-                                                    obj->num_sg,
-                                                    obj->gtt_space->start >> PAGE_SHIFT,
-                                                    agp_type);
-                } else
-                        intel_gtt_insert_pages(obj->gtt_space->start
-                                                   >> PAGE_SHIFT,
-                                               obj->base.size >> PAGE_SHIFT,
-                                               obj->pages,
-                                               agp_type);
+                i915_gem_gtt_rebind_object(obj, obj->cache_level);
         }
 
         intel_gtt_chipset_flush();
@@ -110,6 +94,27 @@ int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
         return 0;
 }
 
+void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
+                                enum i915_cache_level cache_level)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
+
+        if (dev_priv->mm.gtt->needs_dmar) {
+                BUG_ON(!obj->sg_list);
+
+                intel_gtt_insert_sg_entries(obj->sg_list,
+                                            obj->num_sg,
+                                            obj->gtt_space->start >> PAGE_SHIFT,
+                                            agp_type);
+        } else
+                intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
+                                       obj->base.size >> PAGE_SHIFT,
+                                       obj->pages,
+                                       agp_type);
+}
+
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 {
         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 3b03f85ea62..73248d0ea17 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -306,12 +306,15 @@ static void i915_hotplug_work_func(struct work_struct *work)
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *encoder;
 
+        mutex_lock(&mode_config->mutex);
         DRM_DEBUG_KMS("running encoder hotplug functions\n");
 
         list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
                 if (encoder->hot_plug)
                         encoder->hot_plug(encoder);
 
+        mutex_unlock(&mode_config->mutex);
+
         /* Just fire off a uevent and let userspace tell us what to do */
         drm_helper_hpd_irq_event(dev);
 }
@@ -361,10 +364,12 @@ static void notify_ring(struct drm_device *dev,
 
         ring->irq_seqno = seqno;
         wake_up_all(&ring->irq_queue);
-
-        dev_priv->hangcheck_count = 0;
-        mod_timer(&dev_priv->hangcheck_timer,
-                  jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+        if (i915_enable_hangcheck) {
+                dev_priv->hangcheck_count = 0;
+                mod_timer(&dev_priv->hangcheck_timer,
+                          jiffies +
+                          msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+        }
 }
 
 static void gen6_pm_rps_work(struct work_struct *work)
@@ -817,6 +822,7 @@ static void i915_gem_record_fences(struct drm_device *dev,
 
         /* Fences */
         switch (INTEL_INFO(dev)->gen) {
+        case 7:
         case 6:
                 for (i = 0; i < 16; i++)
                         error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
@@ -1664,6 +1670,9 @@ void i915_hangcheck_elapsed(unsigned long data)
         uint32_t acthd, instdone, instdone1;
         bool err = false;
 
+        if (!i915_enable_hangcheck)
+                return;
+
         /* If all work is done then ACTHD clearly hasn't advanced. */
         if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
             i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
@@ -2050,8 +2059,10 @@ void intel_irq_init(struct drm_device *dev)
                 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
         }
 
-
-        dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
+                dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
+        else
+                dev->driver->get_vblank_timestamp = NULL;
         dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
 
         if (IS_IVYBRIDGE(dev)) {
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 5d5def756c9..2ae29de4172 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -78,6 +78,14 @@
 #define  GRDOM_RENDER   (1<<2)
 #define  GRDOM_MEDIA    (3<<2)
 
+#define GEN6_MBCUNIT_SNPCR      0x900c /* for LLC config */
+#define   GEN6_MBC_SNPCR_SHIFT  21
+#define   GEN6_MBC_SNPCR_MASK   (3<<21)
+#define   GEN6_MBC_SNPCR_MAX    (0<<21)
+#define   GEN6_MBC_SNPCR_MED    (1<<21)
+#define   GEN6_MBC_SNPCR_LOW    (2<<21)
+#define   GEN6_MBC_SNPCR_MIN    (3<<21) /* only 1/16th of the cache is shared */
+
 #define GEN6_GDRST      0x941c
 #define  GEN6_GRDOM_FULL                (1 << 0)
 #define  GEN6_GRDOM_RENDER              (1 << 1)
@@ -367,6 +375,7 @@
 # define MI_FLUSH_ENABLE                                (1 << 11)
 
 #define GFX_MODE        0x02520
+#define GFX_MODE_GEN7   0x0229c
 #define   GFX_RUN_LIST_ENABLE           (1<<15)
 #define   GFX_TLB_INVALIDATE_ALWAYS     (1<<13)
 #define   GFX_SURFACE_FAULT_ENABLE      (1<<12)
@@ -374,6 +383,9 @@
 #define   GFX_PSMI_GRANULARITY          (1<<10)
 #define   GFX_PPGTT_ENABLE              (1<<9)
 
+#define GFX_MODE_ENABLE(bit) (((bit) << 16) | (bit))
+#define GFX_MODE_DISABLE(bit) (((bit) << 16) | (0))
+
 #define SCPD0           0x0209c /* 915+ only */
 #define IER             0x020a0
 #define IIR             0x020a4
@@ -579,6 +591,7 @@
 #define   DPFC_CTL_PLANEA       (0<<30)
 #define   DPFC_CTL_PLANEB       (1<<30)
 #define   DPFC_CTL_FENCE_EN     (1<<29)
+#define   DPFC_CTL_PERSISTENT_MODE      (1<<25)
 #define   DPFC_SR_EN            (1<<10)
 #define   DPFC_CTL_LIMIT_1X     (0<<6)
 #define   DPFC_CTL_LIMIT_2X     (1<<6)
@@ -1309,6 +1322,7 @@
 #define   ADPA_PIPE_SELECT_MASK (1<<30)
 #define   ADPA_PIPE_A_SELECT    0
 #define   ADPA_PIPE_B_SELECT    (1<<30)
+#define   ADPA_PIPE_SELECT(pipe) ((pipe) << 30)
 #define   ADPA_USE_VGA_HVPOLARITY (1<<15)
 #define   ADPA_SETS_HVPOLARITY  0
 #define   ADPA_VSYNC_CNTL_DISABLE (1<<11)
@@ -1451,6 +1465,7 @@
 /* Selects pipe B for LVDS data.  Must be set on pre-965. */
 #define   LVDS_PIPEB_SELECT             (1 << 30)
 #define   LVDS_PIPE_MASK                (1 << 30)
+#define   LVDS_PIPE(pipe)               ((pipe) << 30)
 /* LVDS dithering flag on 965/g4x platform */
 #define   LVDS_ENABLE_DITHER            (1 << 25)
 /* LVDS sync polarity flags. Set to invert (i.e. negative) */
@@ -1490,9 +1505,6 @@
 #define   LVDS_B0B3_POWER_DOWN          (0 << 2)
 #define   LVDS_B0B3_POWER_UP            (3 << 2)
 
-#define LVDS_PIPE_ENABLED(V, P) \
-        (((V) & (LVDS_PIPE_MASK | LVDS_PORT_EN)) == ((P) << 30 | LVDS_PORT_EN))
-
 /* Video Data Island Packet control */
 #define VIDEO_DIP_DATA          0x61178
 #define VIDEO_DIP_CTL           0x61170
@@ -1505,6 +1517,7 @@
 #define   VIDEO_DIP_SELECT_AVI          (0 << 19)
 #define   VIDEO_DIP_SELECT_VENDOR       (1 << 19)
 #define   VIDEO_DIP_SELECT_SPD          (3 << 19)
+#define   VIDEO_DIP_SELECT_MASK         (3 << 19)
 #define   VIDEO_DIP_FREQ_ONCE           (0 << 16)
 #define   VIDEO_DIP_FREQ_VSYNC          (1 << 16)
 #define   VIDEO_DIP_FREQ_2VSYNC         (2 << 16)
@@ -2083,9 +2096,6 @@
 #define   DP_PIPEB_SELECT               (1 << 30)
 #define   DP_PIPE_MASK                  (1 << 30)
 
-#define DP_PIPE_ENABLED(V, P) \
-        (((V) & (DP_PIPE_MASK | DP_PORT_EN)) == ((P) << 30 | DP_PORT_EN))
-
 /* Link training mode - select a suitable mode for each stage */
 #define   DP_LINK_TRAIN_PAT_1           (0 << 28)
 #define   DP_LINK_TRAIN_PAT_2           (1 << 28)
@@ -2544,10 +2554,18 @@
 #define _CURBBASE               0x700c4
 #define _CURBPOS                        0x700c8
 
+#define _CURBCNTR_IVB           0x71080
+#define _CURBBASE_IVB           0x71084
+#define _CURBPOS_IVB            0x71088
+
 #define CURCNTR(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR)
 #define CURBASE(pipe) _PIPE(pipe, _CURABASE, _CURBBASE)
 #define CURPOS(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS)
 
+#define CURCNTR_IVB(pipe) _PIPE(pipe, _CURACNTR, _CURBCNTR_IVB)
+#define CURBASE_IVB(pipe) _PIPE(pipe, _CURABASE, _CURBBASE_IVB)
+#define CURPOS_IVB(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS_IVB)
+
 /* Display A control */
 #define _DSPACNTR                0x70180
 #define   DISPLAY_PLANE_ENABLE                  (1<<31)
@@ -3023,6 +3041,20 @@
 #define _TRANSA_DP_LINK_M2       0xe0048
 #define _TRANSA_DP_LINK_N2       0xe004c
 
+/* Per-transcoder DIP controls */
+
+#define _VIDEO_DIP_CTL_A         0xe0200
+#define _VIDEO_DIP_DATA_A        0xe0208
+#define _VIDEO_DIP_GCP_A         0xe0210
+
+#define _VIDEO_DIP_CTL_B         0xe1200
+#define _VIDEO_DIP_DATA_B        0xe1208
+#define _VIDEO_DIP_GCP_B         0xe1210
+
+#define TVIDEO_DIP_CTL(pipe) _PIPE(pipe, _VIDEO_DIP_CTL_A, _VIDEO_DIP_CTL_B)
+#define TVIDEO_DIP_DATA(pipe) _PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B)
+#define TVIDEO_DIP_GCP(pipe) _PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B)
+
 #define _TRANS_HTOTAL_B          0xe1000
 #define _TRANS_HBLANK_B          0xe1004
 #define _TRANS_HSYNC_B           0xe1008
@@ -3075,6 +3107,16 @@
 #define  TRANS_6BPC             (2<<5)
 #define  TRANS_12BPC            (3<<5)
 
+#define _TRANSA_CHICKEN2         0xf0064
+#define _TRANSB_CHICKEN2         0xf1064
+#define TRANS_CHICKEN2(pipe) _PIPE(pipe, _TRANSA_CHICKEN2, _TRANSB_CHICKEN2)
+#define   TRANS_AUTOTRAIN_GEN_STALL_DIS (1<<31)
+
+#define SOUTH_CHICKEN1          0xc2000
+#define  FDIA_PHASE_SYNC_SHIFT_OVR      19
+#define  FDIA_PHASE_SYNC_SHIFT_EN       18
+#define FDI_PHASE_SYNC_OVR(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_OVR - ((pipe) * 2)))
+#define FDI_PHASE_SYNC_EN(pipe) (1<<(FDIA_PHASE_SYNC_SHIFT_EN - ((pipe) * 2)))
 #define SOUTH_CHICKEN2          0xc2004
 #define  DPLS_EDP_PPS_FIX_DIS   (1<<0)
 
@@ -3133,6 +3175,7 @@
 #define  FDI_LINK_TRAIN_NONE_IVB            (3<<8)
 
 /* both Tx and Rx */
+#define  FDI_COMPOSITE_SYNC             (1<<11)
 #define  FDI_LINK_TRAIN_AUTO            (1<<10)
 #define  FDI_SCRAMBLING_ENABLE          (0<<7)
 #define  FDI_SCRAMBLING_DISABLE         (1<<7)
@@ -3225,15 +3268,13 @@
 #define  ADPA_CRT_HOTPLUG_VOLREF_475MV  (1<<17)
 #define  ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1<<16)
 
-#define ADPA_PIPE_ENABLED(V, P) \
-        (((V) & (ADPA_TRANS_SELECT_MASK | ADPA_DAC_ENABLE)) == ((P) << 30 | ADPA_DAC_ENABLE))
-
 /* or SDVOB */
 #define HDMIB   0xe1140
 #define  PORT_ENABLE    (1 << 31)
-#define  TRANSCODER_A   (0)
-#define  TRANSCODER_B   (1 << 30)
-#define  TRANSCODER_MASK   (1 << 30)
+#define  TRANSCODER(pipe)       ((pipe) << 30)
+#define  TRANSCODER_CPT(pipe)   ((pipe) << 29)
+#define  TRANSCODER_MASK        (1 << 30)
+#define  TRANSCODER_MASK_CPT    (3 << 29)
 #define  COLOR_FORMAT_8bpc      (0)
 #define  COLOR_FORMAT_12bpc     (3 << 26)
 #define  SDVOB_HOTPLUG_ENABLE   (1 << 23)
@@ -3249,9 +3290,6 @@
 #define  HSYNC_ACTIVE_HIGH      (1 << 3)
 #define  PORT_DETECTED          (1 << 2)
 
-#define HDMI_PIPE_ENABLED(V, P) \
-        (((V) & (TRANSCODER_MASK | PORT_ENABLE)) == ((P) << 30 | PORT_ENABLE))
-
 /* PCH SDVOB multiplex with HDMIB */
 #define PCH_SDVOB       HDMIB
 
@@ -3318,6 +3356,7 @@
 #define  PORT_TRANS_B_SEL_CPT   (1<<29)
 #define  PORT_TRANS_C_SEL_CPT   (2<<29)
 #define  PORT_TRANS_SEL_MASK    (3<<29)
+#define  PORT_TRANS_SEL_CPT(pipe)       ((pipe) << 29)
 
 #define TRANS_DP_CTL_A          0xe0300
 #define TRANS_DP_CTL_B          0xe1300
@@ -3360,6 +3399,11 @@
 #define  FORCEWAKE_ACK                          0x130090
 
 #define  GT_FIFO_FREE_ENTRIES                   0x120008
+#define    GT_FIFO_NUM_RESERVED_ENTRIES         20
+
+#define GEN6_UCGCTL2                            0x9404
+# define GEN6_RCPBUNIT_CLOCK_GATE_DISABLE               (1 << 12)
+# define GEN6_RCCUNIT_CLOCK_GATE_DISABLE                (1 << 11)
 
 #define GEN6_RPNSWREQ                           0xA008
 #define   GEN6_TURBO_DISABLE                    (1<<31)
@@ -3434,7 +3478,9 @@
 #define GEN6_PCODE_MAILBOX                      0x138124
 #define   GEN6_PCODE_READY                      (1<<31)
 #define   GEN6_READ_OC_PARAMS                   0xc
-#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE       0x9
+#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE       0x8
+#define   GEN6_PCODE_READ_MIN_FREQ_TABLE        0x9
 #define GEN6_PCODE_DATA                         0x138128
+#define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT        8
 
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 5257cfc34c3..9c7706a4c71 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -370,6 +370,7 @@ static void i915_save_modeset_reg(struct drm_device *dev)
 
         /* Fences */
         switch (INTEL_INFO(dev)->gen) {
+        case 7:
         case 6:
                 for (i = 0; i < 16; i++)
                         dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
@@ -404,6 +405,7 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
 
         /* Fences */
         switch (INTEL_INFO(dev)->gen) {
+        case 7:
         case 6:
                 for (i = 0; i < 16; i++)
                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
@@ -760,15 +762,13 @@ static void i915_restore_display(struct drm_device *dev)
         /* FIXME: restore TV & SDVO state */
 
         /* only restore FBC info on the platform that supports FBC*/
+        intel_disable_fbc(dev);
         if (I915_HAS_FBC(dev)) {
                 if (HAS_PCH_SPLIT(dev)) {
-                        ironlake_disable_fbc(dev);
                         I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->saveDPFC_CB_BASE);
                 } else if (IS_GM45(dev)) {
-                        g4x_disable_fbc(dev);
                         I915_WRITE(DPFC_CB_BASE, dev_priv->saveDPFC_CB_BASE);
                 } else {
-                        i8xx_disable_fbc(dev);
                         I915_WRITE(FBC_CFB_BASE, dev_priv->saveFBC_CFB_BASE);
                         I915_WRITE(FBC_LL_BASE, dev_priv->saveFBC_LL_BASE);
                         I915_WRITE(FBC_CONTROL2, dev_priv->saveFBC_CONTROL2);
@@ -814,6 +814,7 @@ int i915_save_state(struct drm_device *dev)
                 dev_priv->saveFDI_RXB_IMR = I915_READ(_FDI_RXB_IMR);
                 dev_priv->saveMCHBAR_RENDER_STANDBY =
                         I915_READ(RSTDBYCTL);
+                dev_priv->savePCH_PORT_HOTPLUG = I915_READ(PCH_PORT_HOTPLUG);
         } else {
                 dev_priv->saveIER = I915_READ(IER);
                 dev_priv->saveIMR = I915_READ(IMR);
@@ -865,21 +866,25 @@ int i915_restore_state(struct drm_device *dev)
                 I915_WRITE(GTIMR, dev_priv->saveGTIMR);
                 I915_WRITE(_FDI_RXA_IMR, dev_priv->saveFDI_RXA_IMR);
                 I915_WRITE(_FDI_RXB_IMR, dev_priv->saveFDI_RXB_IMR);
+                I915_WRITE(PCH_PORT_HOTPLUG, dev_priv->savePCH_PORT_HOTPLUG);
         } else {
                 I915_WRITE(IER, dev_priv->saveIER);
                 I915_WRITE(IMR, dev_priv->saveIMR);
         }
         mutex_unlock(&dev->struct_mutex);
 
-        intel_init_clock_gating(dev);
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
+                intel_init_clock_gating(dev);
 
         if (IS_IRONLAKE_M(dev)) {
                 ironlake_enable_drps(dev);
                 intel_init_emon(dev);
         }
 
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev)) {
                 gen6_enable_rps(dev_priv);
+                gen6_update_ring_freq(dev_priv);
+        }
 
         mutex_lock(&dev->struct_mutex);
 
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 927442a1192..61abef8a811 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -74,7 +74,7 @@ get_blocksize(void *p)
 
 static void
 fill_detail_timing_data(struct drm_display_mode *panel_fixed_mode,
-                        struct lvds_dvo_timing *dvo_timing)
+                        const struct lvds_dvo_timing *dvo_timing)
 {
         panel_fixed_mode->hdisplay = (dvo_timing->hactive_hi << 8) |
                 dvo_timing->hactive_lo;
@@ -115,20 +115,75 @@ fill_detail_timing_data(struct drm_display_mode *panel_fixed_mode,
         drm_mode_set_name(panel_fixed_mode);
 }
 
+static bool
+lvds_dvo_timing_equal_size(const struct lvds_dvo_timing *a,
+                           const struct lvds_dvo_timing *b)
+{
+        if (a->hactive_hi != b->hactive_hi ||
+            a->hactive_lo != b->hactive_lo)
+                return false;
+
+        if (a->hsync_off_hi != b->hsync_off_hi ||
+            a->hsync_off_lo != b->hsync_off_lo)
+                return false;
+
+        if (a->hsync_pulse_width != b->hsync_pulse_width)
+                return false;
+
+        if (a->hblank_hi != b->hblank_hi ||
+            a->hblank_lo != b->hblank_lo)
+                return false;
+
+        if (a->vactive_hi != b->vactive_hi ||
+            a->vactive_lo != b->vactive_lo)
+                return false;
+
+        if (a->vsync_off != b->vsync_off)
+                return false;
+
+        if (a->vsync_pulse_width != b->vsync_pulse_width)
+                return false;
+
+        if (a->vblank_hi != b->vblank_hi ||
+            a->vblank_lo != b->vblank_lo)
+                return false;
+
+        return true;
+}
+
+static const struct lvds_dvo_timing *
+get_lvds_dvo_timing(const struct bdb_lvds_lfp_data *lvds_lfp_data,
+                    const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs,
+                    int index)
+{
+        /*
+         * the size of fp_timing varies on the different platform.
+         * So calculate the DVO timing relative offset in LVDS data
+         * entry to get the DVO timing entry
+         */
+
+        int lfp_data_size =
+                lvds_lfp_data_ptrs->ptr[1].dvo_timing_offset -
+                lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset;
+        int dvo_timing_offset =
+                lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset -
+                lvds_lfp_data_ptrs->ptr[0].fp_timing_offset;
+        char *entry = (char *)lvds_lfp_data->data + lfp_data_size * index;
+
+        return (struct lvds_dvo_timing *)(entry + dvo_timing_offset);
+}
+
 /* Try to find integrated panel data */
 static void
 parse_lfp_panel_data(struct drm_i915_private *dev_priv,
                             struct bdb_header *bdb)
 {
-        struct bdb_lvds_options *lvds_options;
-        struct bdb_lvds_lfp_data *lvds_lfp_data;
-        struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs;
-        struct bdb_lvds_lfp_data_entry *entry;
-        struct lvds_dvo_timing *dvo_timing;
+        const struct bdb_lvds_options *lvds_options;
+        const struct bdb_lvds_lfp_data *lvds_lfp_data;
+        const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs;
+        const struct lvds_dvo_timing *panel_dvo_timing;
         struct drm_display_mode *panel_fixed_mode;
-        int lfp_data_size, dvo_timing_offset;
-        int i, temp_downclock;
-        struct drm_display_mode *temp_mode;
+        int i, downclock;
 
         lvds_options = find_section(bdb, BDB_LVDS_OPTIONS);
         if (!lvds_options)
@@ -150,75 +205,44 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
 
         dev_priv->lvds_vbt = 1;
 
-        lfp_data_size = lvds_lfp_data_ptrs->ptr[1].dvo_timing_offset -
-                lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset;
-        entry = (struct bdb_lvds_lfp_data_entry *)
-                ((uint8_t *)lvds_lfp_data->data + (lfp_data_size *
-                                                   lvds_options->panel_type));
-        dvo_timing_offset = lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset -
-                lvds_lfp_data_ptrs->ptr[0].fp_timing_offset;
-
-        /*
-         * the size of fp_timing varies on the different platform.
-         * So calculate the DVO timing relative offset in LVDS data
-         * entry to get the DVO timing entry
-         */
-        dvo_timing = (struct lvds_dvo_timing *)
-                        ((unsigned char *)entry + dvo_timing_offset);
+        panel_dvo_timing = get_lvds_dvo_timing(lvds_lfp_data,
+                                               lvds_lfp_data_ptrs,
+                                               lvds_options->panel_type);
 
         panel_fixed_mode = kzalloc(sizeof(*panel_fixed_mode), GFP_KERNEL);
         if (!panel_fixed_mode)
                 return;
 
-        fill_detail_timing_data(panel_fixed_mode, dvo_timing);
+        fill_detail_timing_data(panel_fixed_mode, panel_dvo_timing);
 
         dev_priv->lfp_lvds_vbt_mode = panel_fixed_mode;
 
         DRM_DEBUG_KMS("Found panel mode in BIOS VBT tables:\n");
         drm_mode_debug_printmodeline(panel_fixed_mode);
 
-        temp_mode = kzalloc(sizeof(*temp_mode), GFP_KERNEL);
-        temp_downclock = panel_fixed_mode->clock;
         /*
-         * enumerate the LVDS panel timing info entry in VBT to check whether
-         * the LVDS downclock is found.
+         * Iterate over the LVDS panel timing info to find the lowest clock
+         * for the native resolution.
          */
+        downclock = panel_dvo_timing->clock;
         for (i = 0; i < 16; i++) {
-                entry = (struct bdb_lvds_lfp_data_entry *)
-                        ((uint8_t *)lvds_lfp_data->data + (lfp_data_size * i));
-                dvo_timing = (struct lvds_dvo_timing *)
-                        ((unsigned char *)entry + dvo_timing_offset);
-
-                fill_detail_timing_data(temp_mode, dvo_timing);
-
-                if (temp_mode->hdisplay == panel_fixed_mode->hdisplay &&
-                temp_mode->hsync_start == panel_fixed_mode->hsync_start &&
-                temp_mode->hsync_end == panel_fixed_mode->hsync_end &&
-                temp_mode->htotal == panel_fixed_mode->htotal &&
-                temp_mode->vdisplay == panel_fixed_mode->vdisplay &&
-                temp_mode->vsync_start == panel_fixed_mode->vsync_start &&
-                temp_mode->vsync_end == panel_fixed_mode->vsync_end &&
-                temp_mode->vtotal == panel_fixed_mode->vtotal &&
-                temp_mode->clock < temp_downclock) {
-                        /*
-                         * downclock is already found. But we expect
-                         * to find the lower downclock.
-                         */
-                        temp_downclock = temp_mode->clock;
-                }
-                /* clear it to zero */
-                memset(temp_mode, 0, sizeof(*temp_mode));
+                const struct lvds_dvo_timing *dvo_timing;
+
+                dvo_timing = get_lvds_dvo_timing(lvds_lfp_data,
+                                                 lvds_lfp_data_ptrs,
+                                                 i);
+                if (lvds_dvo_timing_equal_size(dvo_timing, panel_dvo_timing) &&
+                    dvo_timing->clock < downclock)
+                        downclock = dvo_timing->clock;
         }
-        kfree(temp_mode);
-        if (temp_downclock < panel_fixed_mode->clock &&
-            i915_lvds_downclock) {
+
+        if (downclock < panel_dvo_timing->clock && i915_lvds_downclock) {
                 dev_priv->lvds_downclock_avail = 1;
-                dev_priv->lvds_downclock = temp_downclock;
+                dev_priv->lvds_downclock = downclock * 10;
                 DRM_DEBUG_KMS("LVDS downclock is found in VBT. "
                               "Normal Clock %dKHz, downclock %dKHz\n",
-                              temp_downclock, panel_fixed_mode->clock);
+                              panel_fixed_mode->clock, 10*downclock);
         }
-        return;
 }
 
 /* Try to find sdvo panel data */
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 0f1c799afea..07e7cf38068 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -24,6 +24,7 @@
  *      Eric Anholt <eric@anholt.net>
  */
 
+#include <linux/cpufreq.h>
 #include <linux/module.h>
 #include <linux/input.h>
 #include <linux/i2c.h>
@@ -877,7 +878,7 @@ static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
         int pp_reg, lvds_reg;
         u32 val;
         enum pipe panel_pipe = PIPE_A;
-        bool locked = locked;
+        bool locked = true;
 
         if (HAS_PCH_SPLIT(dev_priv->dev)) {
                 pp_reg = PCH_PP_CONTROL;
@@ -979,11 +980,76 @@ static void assert_transcoder_disabled(struct drm_i915_private *dev_priv,
              pipe_name(pipe));
 }
 
+static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
+                            enum pipe pipe, u32 port_sel, u32 val)
+{
+        if ((val & DP_PORT_EN) == 0)
+                return false;
+
+        if (HAS_PCH_CPT(dev_priv->dev)) {
+                u32     trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
+                u32     trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
+                if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
+                        return false;
+        } else {
+                if ((val & DP_PIPE_MASK) != (pipe << 30))
+                        return false;
+        }
+        return true;
+}
+
+static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
+                              enum pipe pipe, u32 val)
+{
+        if ((val & PORT_ENABLE) == 0)
+                return false;
+
+        if (HAS_PCH_CPT(dev_priv->dev)) {
+                if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
+                        return false;
+        } else {
+                if ((val & TRANSCODER_MASK) != TRANSCODER(pipe))
+                        return false;
+        }
+        return true;
+}
+
+static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
+                              enum pipe pipe, u32 val)
+{
+        if ((val & LVDS_PORT_EN) == 0)
+                return false;
+
+        if (HAS_PCH_CPT(dev_priv->dev)) {
+                if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
+                        return false;
+        } else {
+                if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
+                        return false;
+        }
+        return true;
+}
+
+static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
+                              enum pipe pipe, u32 val)
+{
+        if ((val & ADPA_DAC_ENABLE) == 0)
+                return false;
+        if (HAS_PCH_CPT(dev_priv->dev)) {
+                if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
+                        return false;
+        } else {
+                if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
+                        return false;
+        }
+        return true;
+}
+
 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
-                                   enum pipe pipe, int reg)
+                                   enum pipe pipe, int reg, u32 port_sel)
 {
         u32 val = I915_READ(reg);
-        WARN(DP_PIPE_ENABLED(val, pipe),
+        WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
              reg, pipe_name(pipe));
 }
@@ -992,7 +1058,7 @@ static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
                                      enum pipe pipe, int reg)
 {
         u32 val = I915_READ(reg);
-        WARN(HDMI_PIPE_ENABLED(val, pipe),
+        WARN(hdmi_pipe_enabled(dev_priv, val, pipe),
              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
              reg, pipe_name(pipe));
 }
@@ -1003,19 +1069,19 @@ static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
         int reg;
         u32 val;
 
-        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B);
-        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C);
-        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D);
+        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
+        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
+        assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
 
         reg = PCH_ADPA;
         val = I915_READ(reg);
-        WARN(ADPA_PIPE_ENABLED(val, pipe),
+        WARN(adpa_pipe_enabled(dev_priv, val, pipe),
              "PCH VGA enabled on transcoder %c, should be disabled\n",
              pipe_name(pipe));
 
         reg = PCH_LVDS;
         val = I915_READ(reg);
-        WARN(LVDS_PIPE_ENABLED(val, pipe),
+        WARN(lvds_pipe_enabled(dev_priv, val, pipe),
              "PCH LVDS enabled on transcoder %c, should be disabled\n",
              pipe_name(pipe));
 
@@ -1157,12 +1223,15 @@ static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
 
         reg = TRANSCONF(pipe);
         val = I915_READ(reg);
-        /*
-         * make the BPC in transcoder be consistent with
-         * that in pipeconf reg.
-         */
-        val &= ~PIPE_BPC_MASK;
-        val |= I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK;
+
+        if (HAS_PCH_IBX(dev_priv->dev)) {
+                /*
+                 * make the BPC in transcoder be consistent with
+                 * that in pipeconf reg.
+                 */
+                val &= ~PIPE_BPC_MASK;
+                val |= I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK;
+        }
         I915_WRITE(reg, val | TRANS_ENABLE);
         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
                 DRM_ERROR("failed to enable transcoder %d\n", pipe);
@@ -1272,6 +1341,17 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv,
         intel_wait_for_pipe_off(dev_priv->dev, pipe);
 }
 
+/*
+ * Plane regs are double buffered, going from enabled->disabled needs a
+ * trigger in order to latch.  The display address reg provides this.
+ */
+static void intel_flush_display_plane(struct drm_i915_private *dev_priv,
+                                      enum plane plane)
+{
+        I915_WRITE(DSPADDR(plane), I915_READ(DSPADDR(plane)));
+        I915_WRITE(DSPSURF(plane), I915_READ(DSPSURF(plane)));
+}
+
 /**
  * intel_enable_plane - enable a display plane on a given pipe
  * @dev_priv: i915 private structure
@@ -1295,20 +1375,10 @@ static void intel_enable_plane(struct drm_i915_private *dev_priv,
                 return;
 
         I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
+        intel_flush_display_plane(dev_priv, plane);
         intel_wait_for_vblank(dev_priv->dev, pipe);
 }
 
-/*
- * Plane regs are double buffered, going from enabled->disabled needs a
- * trigger in order to latch.  The display address reg provides this.
- */
-static void intel_flush_display_plane(struct drm_i915_private *dev_priv,
-                                      enum plane plane)
-{
-        u32 reg = DSPADDR(plane);
-        I915_WRITE(reg, I915_READ(reg));
-}
-
 /**
  * intel_disable_plane - disable a display plane
  * @dev_priv: i915 private structure
@@ -1334,19 +1404,24 @@ static void intel_disable_plane(struct drm_i915_private *dev_priv,
 }
 
 static void disable_pch_dp(struct drm_i915_private *dev_priv,
-                           enum pipe pipe, int reg)
+                           enum pipe pipe, int reg, u32 port_sel)
 {
         u32 val = I915_READ(reg);
-        if (DP_PIPE_ENABLED(val, pipe))
+        if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) {
+                DRM_DEBUG_KMS("Disabling pch dp %x on pipe %d\n", reg, pipe);
                 I915_WRITE(reg, val & ~DP_PORT_EN);
+        }
 }
 
 static void disable_pch_hdmi(struct drm_i915_private *dev_priv,
                              enum pipe pipe, int reg)
 {
         u32 val = I915_READ(reg);
-        if (HDMI_PIPE_ENABLED(val, pipe))
+        if (hdmi_pipe_enabled(dev_priv, val, pipe)) {
+                DRM_DEBUG_KMS("Disabling pch HDMI %x on pipe %d\n",
+                              reg, pipe);
                 I915_WRITE(reg, val & ~PORT_ENABLE);
+        }
 }
 
 /* Disable any ports connected to this transcoder */
@@ -1358,18 +1433,19 @@ static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
         val = I915_READ(PCH_PP_CONTROL);
         I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS);
 
-        disable_pch_dp(dev_priv, pipe, PCH_DP_B);
-        disable_pch_dp(dev_priv, pipe, PCH_DP_C);
-        disable_pch_dp(dev_priv, pipe, PCH_DP_D);
+        disable_pch_dp(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
+        disable_pch_dp(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
+        disable_pch_dp(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
 
         reg = PCH_ADPA;
         val = I915_READ(reg);
-        if (ADPA_PIPE_ENABLED(val, pipe))
+        if (adpa_pipe_enabled(dev_priv, val, pipe))
                 I915_WRITE(reg, val & ~ADPA_DAC_ENABLE);
 
         reg = PCH_LVDS;
         val = I915_READ(reg);
-        if (LVDS_PIPE_ENABLED(val, pipe)) {
+        if (lvds_pipe_enabled(dev_priv, val, pipe)) {
+                DRM_DEBUG_KMS("disable lvds on pipe %d val 0x%08x\n", pipe, val);
                 I915_WRITE(reg, val & ~LVDS_PORT_EN);
                 POSTING_READ(reg);
                 udelay(100);
@@ -1380,6 +1456,28 @@ static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
         disable_pch_hdmi(dev_priv, pipe, HDMID);
 }
 
+static void i8xx_disable_fbc(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 fbc_ctl;
+
+        /* Disable compression */
+        fbc_ctl = I915_READ(FBC_CONTROL);
+        if ((fbc_ctl & FBC_CTL_EN) == 0)
+                return;
+
+        fbc_ctl &= ~FBC_CTL_EN;
+        I915_WRITE(FBC_CONTROL, fbc_ctl);
+
+        /* Wait for compressing bit to clear */
+        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
+                DRM_DEBUG_KMS("FBC idle timed out\n");
+                return;
+        }
+
+        DRM_DEBUG_KMS("disabled FBC\n");
+}
+
 static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 {
         struct drm_device *dev = crtc->dev;
@@ -1388,36 +1486,25 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int cfb_pitch;
         int plane, i;
         u32 fbc_ctl, fbc_ctl2;
 
-        if (fb->pitch == dev_priv->cfb_pitch &&
-            obj->fence_reg == dev_priv->cfb_fence &&
-            intel_crtc->plane == dev_priv->cfb_plane &&
-            I915_READ(FBC_CONTROL) & FBC_CTL_EN)
-                return;
-
-        i8xx_disable_fbc(dev);
-
-        dev_priv->cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
-
-        if (fb->pitch < dev_priv->cfb_pitch)
-                dev_priv->cfb_pitch = fb->pitch;
+        cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
+        if (fb->pitch < cfb_pitch)
+                cfb_pitch = fb->pitch;
 
         /* FBC_CTL wants 64B units */
-        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj->fence_reg;
-        dev_priv->cfb_plane = intel_crtc->plane;
-        plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
+        cfb_pitch = (cfb_pitch / 64) - 1;
+        plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
 
         /* Clear old tags */
         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
                 I915_WRITE(FBC_TAG + (i * 4), 0);
 
         /* Set it up... */
-        fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
-        if (obj->tiling_mode != I915_TILING_NONE)
-                fbc_ctl2 |= FBC_CTL_CPU_FENCE;
+        fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
+        fbc_ctl2 |= plane;
         I915_WRITE(FBC_CONTROL2, fbc_ctl2);
         I915_WRITE(FBC_FENCE_OFF, crtc->y);
 
@@ -1425,36 +1512,13 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
         if (IS_I945GM(dev))
                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
-        fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
+        fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
         fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
-        if (obj->tiling_mode != I915_TILING_NONE)
-                fbc_ctl |= dev_priv->cfb_fence;
-        I915_WRITE(FBC_CONTROL, fbc_ctl);
-
-        DRM_DEBUG_KMS("enabled FBC, pitch %ld, yoff %d, plane %d, ",
-                      dev_priv->cfb_pitch, crtc->y, dev_priv->cfb_plane);
-}
-
-void i8xx_disable_fbc(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 fbc_ctl;
-
-        /* Disable compression */
-        fbc_ctl = I915_READ(FBC_CONTROL);
-        if ((fbc_ctl & FBC_CTL_EN) == 0)
-                return;
-
-        fbc_ctl &= ~FBC_CTL_EN;
+        fbc_ctl |= obj->fence_reg;
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
-        /* Wait for compressing bit to clear */
-        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
-                DRM_DEBUG_KMS("FBC idle timed out\n");
-                return;
-        }
-
-        DRM_DEBUG_KMS("disabled FBC\n");
+        DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
+                      cfb_pitch, crtc->y, intel_crtc->plane);
 }
 
 static bool i8xx_fbc_enabled(struct drm_device *dev)
@@ -1476,30 +1540,9 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         unsigned long stall_watermark = 200;
         u32 dpfc_ctl;
 
-        dpfc_ctl = I915_READ(DPFC_CONTROL);
-        if (dpfc_ctl & DPFC_CTL_EN) {
-                if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
-                    dev_priv->cfb_fence == obj->fence_reg &&
-                    dev_priv->cfb_plane == intel_crtc->plane &&
-                    dev_priv->cfb_y == crtc->y)
-                        return;
-
-                I915_WRITE(DPFC_CONTROL, dpfc_ctl & ~DPFC_CTL_EN);
-                intel_wait_for_vblank(dev, intel_crtc->pipe);
-        }
-
-        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj->fence_reg;
-        dev_priv->cfb_plane = intel_crtc->plane;
-        dev_priv->cfb_y = crtc->y;
-
         dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
-        if (obj->tiling_mode != I915_TILING_NONE) {
-                dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
-                I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
-        } else {
-                I915_WRITE(DPFC_CHICKEN, ~DPFC_HT_MODIFY);
-        }
+        dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
+        I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
 
         I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
@@ -1512,7 +1555,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
 }
 
-void g4x_disable_fbc(struct drm_device *dev)
+static void g4x_disable_fbc(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 dpfc_ctl;
@@ -1567,32 +1610,12 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         u32 dpfc_ctl;
 
         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
-        if (dpfc_ctl & DPFC_CTL_EN) {
-                if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
-                    dev_priv->cfb_fence == obj->fence_reg &&
-                    dev_priv->cfb_plane == intel_crtc->plane &&
-                    dev_priv->cfb_offset == obj->gtt_offset &&
-                    dev_priv->cfb_y == crtc->y)
-                        return;
-
-                I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl & ~DPFC_CTL_EN);
-                intel_wait_for_vblank(dev, intel_crtc->pipe);
-        }
-
-        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj->fence_reg;
-        dev_priv->cfb_plane = intel_crtc->plane;
-        dev_priv->cfb_offset = obj->gtt_offset;
-        dev_priv->cfb_y = crtc->y;
-
         dpfc_ctl &= DPFC_RESERVED;
         dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
-        if (obj->tiling_mode != I915_TILING_NONE) {
-                dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence);
-                I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
-        } else {
-                I915_WRITE(ILK_DPFC_CHICKEN, ~DPFC_HT_MODIFY);
-        }
+        /* Set persistent mode for front-buffer rendering, ala X. */
+        dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
+        dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
+        I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
 
         I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
@@ -1604,7 +1627,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 
         if (IS_GEN6(dev)) {
                 I915_WRITE(SNB_DPFC_CTL_SA,
-                           SNB_CPU_FENCE_ENABLE | dev_priv->cfb_fence);
+                           SNB_CPU_FENCE_ENABLE | obj->fence_reg);
                 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
                 sandybridge_blit_fbc_update(dev);
         }
@@ -1612,7 +1635,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
         DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
 }
 
-void ironlake_disable_fbc(struct drm_device *dev)
+static void ironlake_disable_fbc(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 dpfc_ctl;
@@ -1644,24 +1667,109 @@ bool intel_fbc_enabled(struct drm_device *dev)
         return dev_priv->display.fbc_enabled(dev);
 }
 
-void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+static void intel_fbc_work_fn(struct work_struct *__work)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct intel_fbc_work *work =
+                container_of(to_delayed_work(__work),
+                             struct intel_fbc_work, work);
+        struct drm_device *dev = work->crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        mutex_lock(&dev->struct_mutex);
+        if (work == dev_priv->fbc_work) {
+                /* Double check that we haven't switched fb without cancelling
+                 * the prior work.
+                 */
+                if (work->crtc->fb == work->fb) {
+                        dev_priv->display.enable_fbc(work->crtc,
+                                                     work->interval);
+
+                        dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
+                        dev_priv->cfb_fb = work->crtc->fb->base.id;
+                        dev_priv->cfb_y = work->crtc->y;
+                }
+
+                dev_priv->fbc_work = NULL;
+        }
+        mutex_unlock(&dev->struct_mutex);
+
+        kfree(work);
+}
+
+static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
+{
+        if (dev_priv->fbc_work == NULL)
+                return;
+
+        DRM_DEBUG_KMS("cancelling pending FBC enable\n");
+
+        /* Synchronisation is provided by struct_mutex and checking of
+         * dev_priv->fbc_work, so we can perform the cancellation
+         * entirely asynchronously.
+         */
+        if (cancel_delayed_work(&dev_priv->fbc_work->work))
+                /* tasklet was killed before being run, clean up */
+                kfree(dev_priv->fbc_work);
+
+        /* Mark the work as no longer wanted so that if it does
+         * wake-up (because the work was already running and waiting
+         * for our mutex), it will discover that is no longer
+         * necessary to run.
+         */
+        dev_priv->fbc_work = NULL;
+}
+
+static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+        struct intel_fbc_work *work;
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (!dev_priv->display.enable_fbc)
                 return;
 
-        dev_priv->display.enable_fbc(crtc, interval);
+        intel_cancel_fbc_work(dev_priv);
+
+        work = kzalloc(sizeof *work, GFP_KERNEL);
+        if (work == NULL) {
+                dev_priv->display.enable_fbc(crtc, interval);
+                return;
+        }
+
+        work->crtc = crtc;
+        work->fb = crtc->fb;
+        work->interval = interval;
+        INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
+
+        dev_priv->fbc_work = work;
+
+        DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
+
+        /* Delay the actual enabling to let pageflipping cease and the
+         * display to settle before starting the compression. Note that
+         * this delay also serves a second purpose: it allows for a
+         * vblank to pass after disabling the FBC before we attempt
+         * to modify the control registers.
+         *
+         * A more complicated solution would involve tracking vblanks
+         * following the termination of the page-flipping sequence
+         * and indeed performing the enable as a co-routine and not
+         * waiting synchronously upon the vblank.
+         */
+        schedule_delayed_work(&work->work, msecs_to_jiffies(50));
 }
 
 void intel_disable_fbc(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        intel_cancel_fbc_work(dev_priv);
+
         if (!dev_priv->display.disable_fbc)
                 return;
 
         dev_priv->display.disable_fbc(dev);
+        dev_priv->cfb_plane = -1;
 }
 
 /**
@@ -1691,6 +1799,7 @@ static void intel_update_fbc(struct drm_device *dev)
         struct drm_framebuffer *fb;
         struct intel_framebuffer *intel_fb;
         struct drm_i915_gem_object *obj;
+        int enable_fbc;
 
         DRM_DEBUG_KMS("\n");
 
@@ -1731,8 +1840,15 @@ static void intel_update_fbc(struct drm_device *dev)
         intel_fb = to_intel_framebuffer(fb);
         obj = intel_fb->obj;
 
-        if (!i915_enable_fbc) {
-                DRM_DEBUG_KMS("fbc disabled per module param (default off)\n");
+        enable_fbc = i915_enable_fbc;
+        if (enable_fbc < 0) {
+                DRM_DEBUG_KMS("fbc set to per-chip default\n");
+                enable_fbc = 1;
+                if (INTEL_INFO(dev)->gen <= 5)
+                        enable_fbc = 0;
+        }
+        if (!enable_fbc) {
+                DRM_DEBUG_KMS("fbc disabled per module param\n");
                 dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
                 goto out_disable;
         }
@@ -1760,8 +1876,13 @@ static void intel_update_fbc(struct drm_device *dev)
                 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
                 goto out_disable;
         }
-        if (obj->tiling_mode != I915_TILING_X) {
-                DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n");
+
+        /* The use of a CPU fence is mandatory in order to detect writes
+         * by the CPU to the scanout and trigger updates to the FBC.
+         */
+        if (obj->tiling_mode != I915_TILING_X ||
+            obj->fence_reg == I915_FENCE_REG_NONE) {
+                DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
                 dev_priv->no_fbc_reason = FBC_NOT_TILED;
                 goto out_disable;
         }
@@ -1770,6 +1891,44 @@ static void intel_update_fbc(struct drm_device *dev)
         if (in_dbg_master())
                 goto out_disable;
 
+        /* If the scanout has not changed, don't modify the FBC settings.
+         * Note that we make the fundamental assumption that the fb->obj
+         * cannot be unpinned (and have its GTT offset and fence revoked)
+         * without first being decoupled from the scanout and FBC disabled.
+         */
+        if (dev_priv->cfb_plane == intel_crtc->plane &&
+            dev_priv->cfb_fb == fb->base.id &&
+            dev_priv->cfb_y == crtc->y)
+                return;
+
+        if (intel_fbc_enabled(dev)) {
+                /* We update FBC along two paths, after changing fb/crtc
+                 * configuration (modeswitching) and after page-flipping
+                 * finishes. For the latter, we know that not only did
+                 * we disable the FBC at the start of the page-flip
+                 * sequence, but also more than one vblank has passed.
+                 *
+                 * For the former case of modeswitching, it is possible
+                 * to switch between two FBC valid configurations
+                 * instantaneously so we do need to disable the FBC
+                 * before we can modify its control registers. We also
+                 * have to wait for the next vblank for that to take
+                 * effect. However, since we delay enabling FBC we can
+                 * assume that a vblank has passed since disabling and
+                 * that we can safely alter the registers in the deferred
+                 * callback.
+                 *
+                 * In the scenario that we go from a valid to invalid
+                 * and then back to valid FBC configuration we have
+                 * no strict enforcement that a vblank occurred since
+                 * disabling the FBC. However, along all current pipe
+                 * disabling paths we do need to wait for a vblank at
+                 * some point. And we wait before enabling FBC anyway.
+                 */
+                DRM_DEBUG_KMS("disabling active FBC for update\n");
+                intel_disable_fbc(dev);
+        }
+
         intel_enable_fbc(crtc, 500);
         return;
 
@@ -1812,14 +1971,10 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
         }
 
         dev_priv->mm.interruptible = false;
-        ret = i915_gem_object_pin(obj, alignment, true);
+        ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined);
         if (ret)
                 goto err_interruptible;
 
-        ret = i915_gem_object_set_to_display_plane(obj, pipelined);
-        if (ret)
-                goto err_unpin;
-
         /* Install a fence for tiled scan-out. Pre-i965 always needs a
          * fence, whereas 965+ only requires a fence if using
          * framebuffer compression.  For simplicity, we always install
@@ -1841,10 +1996,8 @@ err_interruptible:
         return ret;
 }
 
-/* Assume fb object is pinned & idle & fenced and just update base pointers */
-static int
-intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-                           int x, int y, enum mode_set_atomic state)
+static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                             int x, int y)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1887,7 +2040,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
                 break;
         default:
-                DRM_ERROR("Unknown color depth\n");
+                DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
                 return -EINVAL;
         }
         if (INTEL_INFO(dev)->gen >= 4) {
@@ -1897,10 +2050,6 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                         dspcntr &= ~DISPPLANE_TILED;
         }
 
-        if (HAS_PCH_SPLIT(dev))
-                /* must disable */
-                dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
-
         I915_WRITE(reg, dspcntr);
 
         Start = obj->gtt_offset;
@@ -1917,6 +2066,99 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 I915_WRITE(DSPADDR(plane), Start + Offset);
         POSTING_READ(reg);
 
+        return 0;
+}
+
+static int ironlake_update_plane(struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb, int x, int y)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_framebuffer *intel_fb;
+        struct drm_i915_gem_object *obj;
+        int plane = intel_crtc->plane;
+        unsigned long Start, Offset;
+        u32 dspcntr;
+        u32 reg;
+
+        switch (plane) {
+        case 0:
+        case 1:
+                break;
+        default:
+                DRM_ERROR("Can't update plane %d in SAREA\n", plane);
+                return -EINVAL;
+        }
+
+        intel_fb = to_intel_framebuffer(fb);
+        obj = intel_fb->obj;
+
+        reg = DSPCNTR(plane);
+        dspcntr = I915_READ(reg);
+        /* Mask out pixel format bits in case we change it */
+        dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
+        switch (fb->bits_per_pixel) {
+        case 8:
+                dspcntr |= DISPPLANE_8BPP;
+                break;
+        case 16:
+                if (fb->depth != 16)
+                        return -EINVAL;
+
+                dspcntr |= DISPPLANE_16BPP;
+                break;
+        case 24:
+        case 32:
+                if (fb->depth == 24)
+                        dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
+                else if (fb->depth == 30)
+                        dspcntr |= DISPPLANE_32BPP_30BIT_NO_ALPHA;
+                else
+                        return -EINVAL;
+                break;
+        default:
+                DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
+                return -EINVAL;
+        }
+
+        if (obj->tiling_mode != I915_TILING_NONE)
+                dspcntr |= DISPPLANE_TILED;
+        else
+                dspcntr &= ~DISPPLANE_TILED;
+
+        /* must disable */
+        dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
+
+        I915_WRITE(reg, dspcntr);
+
+        Start = obj->gtt_offset;
+        Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
+
+        DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
+                      Start, Offset, x, y, fb->pitch);
+        I915_WRITE(DSPSTRIDE(plane), fb->pitch);
+        I915_WRITE(DSPSURF(plane), Start);
+        I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
+        I915_WRITE(DSPADDR(plane), Offset);
+        POSTING_READ(reg);
+
+        return 0;
+}
+
+/* Assume fb object is pinned & idle & fenced and just update base pointers */
+static int
+intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                           int x, int y, enum mode_set_atomic state)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        ret = dev_priv->display.update_plane(crtc, fb, x, y);
+        if (ret)
+                return ret;
+
         intel_update_fbc(dev);
         intel_increase_pllclock(crtc);
 
@@ -1934,7 +2176,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 
         /* no fb bound */
         if (!crtc->fb) {
-                DRM_DEBUG_KMS("No FB bound\n");
+                DRM_ERROR("No FB bound\n");
                 return 0;
         }
 
@@ -1943,6 +2185,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         case 1:
                 break;
         default:
+                DRM_ERROR("no plane for crtc\n");
                 return -EINVAL;
         }
 
@@ -1952,6 +2195,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                                          NULL);
         if (ret != 0) {
                 mutex_unlock(&dev->struct_mutex);
+                DRM_ERROR("pin & fence failed\n");
                 return ret;
         }
 
@@ -1971,7 +2215,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                  * This should only fail upon a hung GPU, in which case we
                  * can safely continue.
                  */
-                ret = i915_gem_object_flush_gpu(obj);
+                ret = i915_gem_object_finish_gpu(obj);
                 (void) ret;
         }
 
@@ -1980,6 +2224,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         if (ret) {
                 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
                 mutex_unlock(&dev->struct_mutex);
+                DRM_ERROR("failed to update base address\n");
                 return ret;
         }
 
@@ -2086,6 +2331,18 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
                            FDI_FE_ERRC_ENABLE);
 }
 
+static void cpt_phase_pointer_enable(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 flags = I915_READ(SOUTH_CHICKEN1);
+
+        flags |= FDI_PHASE_SYNC_OVR(pipe);
+        I915_WRITE(SOUTH_CHICKEN1, flags); /* once to unlock... */
+        flags |= FDI_PHASE_SYNC_EN(pipe);
+        I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to enable */
+        POSTING_READ(SOUTH_CHICKEN1);
+}
+
 /* The FDI link training functions for ILK/Ibexpeak. */
 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
 {
@@ -2236,6 +2493,9 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
         POSTING_READ(reg);
         udelay(150);
 
+        if (HAS_PCH_CPT(dev))
+                cpt_phase_pointer_enable(dev, pipe);
+
         for (i = 0; i < 4; i++ ) {
                 reg = FDI_TX_CTL(pipe);
                 temp = I915_READ(reg);
@@ -2340,6 +2600,7 @@ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
         temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
+        temp |= FDI_COMPOSITE_SYNC;
         I915_WRITE(reg, temp | FDI_TX_ENABLE);
 
         reg = FDI_RX_CTL(pipe);
@@ -2347,11 +2608,15 @@ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
         temp &= ~FDI_LINK_TRAIN_AUTO;
         temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
         temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
+        temp |= FDI_COMPOSITE_SYNC;
         I915_WRITE(reg, temp | FDI_RX_ENABLE);
 
         POSTING_READ(reg);
         udelay(150);
 
+        if (HAS_PCH_CPT(dev))
+                cpt_phase_pointer_enable(dev, pipe);
+
         for (i = 0; i < 4; i++ ) {
                 reg = FDI_TX_CTL(pipe);
                 temp = I915_READ(reg);
@@ -2461,6 +2726,17 @@ static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
         }
 }
 
+static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 flags = I915_READ(SOUTH_CHICKEN1);
+
+        flags &= ~(FDI_PHASE_SYNC_EN(pipe));
+        I915_WRITE(SOUTH_CHICKEN1, flags); /* once to disable... */
+        flags &= ~(FDI_PHASE_SYNC_OVR(pipe));
+        I915_WRITE(SOUTH_CHICKEN1, flags); /* then again to lock */
+        POSTING_READ(SOUTH_CHICKEN1);
+}
 static void ironlake_fdi_disable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
@@ -2490,6 +2766,8 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
                 I915_WRITE(FDI_RX_CHICKEN(pipe),
                            I915_READ(FDI_RX_CHICKEN(pipe) &
                                      ~FDI_RX_PHASE_SYNC_POINTER_EN));
+        } else if (HAS_PCH_CPT(dev)) {
+                cpt_phase_pointer_disable(dev, pipe);
         }
 
         /* still set train pattern 1 */
@@ -2622,6 +2900,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
         /* For PCH DP, enable TRANS_DP_CTL */
         if (HAS_PCH_CPT(dev) &&
             intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
+                u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) >> 5;
                 reg = TRANS_DP_CTL(pipe);
                 temp = I915_READ(reg);
                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
@@ -2629,7 +2908,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
                           TRANS_DP_BPC_MASK);
                 temp |= (TRANS_DP_OUTPUT_ENABLE |
                          TRANS_DP_ENH_FRAMING);
-                temp |= TRANS_DP_8BPC;
+                temp |= bpc << 9; /* same format but at 11:9 */
 
                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
@@ -2699,14 +2978,18 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
                 I915_WRITE(PF_WIN_SZ(pipe), dev_priv->pch_pf_size);
         }
 
+        /*
+         * On ILK+ LUT must be loaded before the pipe is running but with
+         * clocks enabled
+         */
+        intel_crtc_load_lut(crtc);
+
         intel_enable_pipe(dev_priv, pipe, is_pch_port);
         intel_enable_plane(dev_priv, plane, pipe);
 
         if (is_pch_port)
                 ironlake_pch_enable(crtc);
 
-        intel_crtc_load_lut(crtc);
-
         mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
         mutex_unlock(&dev->struct_mutex);
@@ -2732,9 +3015,8 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 
         intel_disable_plane(dev_priv, plane, pipe);
 
-        if (dev_priv->cfb_plane == plane &&
-            dev_priv->display.disable_fbc)
-                dev_priv->display.disable_fbc(dev);
+        if (dev_priv->cfb_plane == plane)
+                intel_disable_fbc(dev);
 
         intel_disable_pipe(dev_priv, pipe);
 
@@ -2898,9 +3180,8 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
         intel_crtc_dpms_overlay(intel_crtc, false);
         intel_crtc_update_cursor(crtc, false);
 
-        if (dev_priv->cfb_plane == plane &&
-            dev_priv->display.disable_fbc)
-                dev_priv->display.disable_fbc(dev);
+        if (dev_priv->cfb_plane == plane)
+                intel_disable_fbc(dev);
 
         intel_disable_plane(dev_priv, plane, pipe);
         intel_disable_pipe(dev_priv, pipe);
@@ -4309,6 +4590,137 @@ static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
 }
 
+/**
+ * intel_choose_pipe_bpp_dither - figure out what color depth the pipe should send
+ * @crtc: CRTC structure
+ *
+ * A pipe may be connected to one or more outputs.  Based on the depth of the
+ * attached framebuffer, choose a good color depth to use on the pipe.
+ *
+ * If possible, match the pipe depth to the fb depth.  In some cases, this
+ * isn't ideal, because the connected output supports a lesser or restricted
+ * set of depths.  Resolve that here:
+ *    LVDS typically supports only 6bpc, so clamp down in that case
+ *    HDMI supports only 8bpc or 12bpc, so clamp to 8bpc with dither for 10bpc
+ *    Displays may support a restricted set as well, check EDID and clamp as
+ *      appropriate.
+ *
+ * RETURNS:
+ * Dithering requirement (i.e. false if display bpc and pipe bpc match,
+ * true if they don't match).
+ */
+static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
+                                         unsigned int *pipe_bpp)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_encoder *encoder;
+        struct drm_connector *connector;
+        unsigned int display_bpc = UINT_MAX, bpc;
+
+        /* Walk the encoders & connectors on this crtc, get min bpc */
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+                struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
+
+                if (encoder->crtc != crtc)
+                        continue;
+
+                if (intel_encoder->type == INTEL_OUTPUT_LVDS) {
+                        unsigned int lvds_bpc;
+
+                        if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) ==
+                            LVDS_A3_POWER_UP)
+                                lvds_bpc = 8;
+                        else
+                                lvds_bpc = 6;
+
+                        if (lvds_bpc < display_bpc) {
+                                DRM_DEBUG_DRIVER("clamping display bpc (was %d) to LVDS (%d)\n", display_bpc, lvds_bpc);
+                                display_bpc = lvds_bpc;
+                        }
+                        continue;
+                }
+
+                if (intel_encoder->type == INTEL_OUTPUT_EDP) {
+                        /* Use VBT settings if we have an eDP panel */
+                        unsigned int edp_bpc = dev_priv->edp.bpp / 3;
+
+                        if (edp_bpc < display_bpc) {
+                                DRM_DEBUG_DRIVER("clamping display bpc (was %d) to eDP (%d)\n", display_bpc, edp_bpc);
+                                display_bpc = edp_bpc;
+                        }
+                        continue;
+                }
+
+                /* Not one of the known troublemakers, check the EDID */
+                list_for_each_entry(connector, &dev->mode_config.connector_list,
+                                    head) {
+                        if (connector->encoder != encoder)
+                                continue;
+
+                        /* Don't use an invalid EDID bpc value */
+                        if (connector->display_info.bpc &&
+                            connector->display_info.bpc < display_bpc) {
+                                DRM_DEBUG_DRIVER("clamping display bpc (was %d) to EDID reported max of %d\n", display_bpc, connector->display_info.bpc);
+                                display_bpc = connector->display_info.bpc;
+                        }
+                }
+
+                /*
+                 * HDMI is either 12 or 8, so if the display lets 10bpc sneak
+                 * through, clamp it down.  (Note: >12bpc will be caught below.)
+                 */
+                if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
+                        if (display_bpc > 8 && display_bpc < 12) {
+                                DRM_DEBUG_DRIVER("forcing bpc to 12 for HDMI\n");
+                                display_bpc = 12;
+                        } else {
+                                DRM_DEBUG_DRIVER("forcing bpc to 8 for HDMI\n");
+                                display_bpc = 8;
+                        }
+                }
+        }
+
+        /*
+         * We could just drive the pipe at the highest bpc all the time and
+         * enable dithering as needed, but that costs bandwidth.  So choose
+         * the minimum value that expresses the full color range of the fb but
+         * also stays within the max display bpc discovered above.
+         */
+
+        switch (crtc->fb->depth) {
+        case 8:
+                bpc = 8; /* since we go through a colormap */
+                break;
+        case 15:
+        case 16:
+                bpc = 6; /* min is 18bpp */
+                break;
+        case 24:
+                bpc = 8;
+                break;
+        case 30:
+                bpc = 10;
+                break;
+        case 48:
+                bpc = 12;
+                break;
+        default:
+                DRM_DEBUG("unsupported depth, assuming 24 bits\n");
+                bpc = min((unsigned int)8, display_bpc);
+                break;
+        }
+
+        display_bpc = min(display_bpc, bpc);
+
+        DRM_DEBUG_DRIVER("setting pipe bpc to %d (max display bpc %d)\n",
+                         bpc, display_bpc);
+
+        *pipe_bpp = display_bpc * 3;
+
+        return display_bpc != bpc;
+}
+
 static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
                               struct drm_display_mode *mode,
                               struct drm_display_mode *adjusted_mode,
@@ -4697,6 +5109,81 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
         return ret;
 }
 
+static void ironlake_update_pch_refclk(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_mode_config *mode_config = &dev->mode_config;
+        struct drm_crtc *crtc;
+        struct intel_encoder *encoder;
+        struct intel_encoder *has_edp_encoder = NULL;
+        u32 temp;
+        bool has_lvds = false;
+
+        /* We need to take the global config into account */
+        list_for_each_entry(crtc, &mode_config->crtc_list, head) {
+                if (!crtc->enabled)
+                        continue;
+
+                list_for_each_entry(encoder, &mode_config->encoder_list,
+                                    base.head) {
+                        if (encoder->base.crtc != crtc)
+                                continue;
+
+                        switch (encoder->type) {
+                        case INTEL_OUTPUT_LVDS:
+                                has_lvds = true;
+                        case INTEL_OUTPUT_EDP:
+                                has_edp_encoder = encoder;
+                                break;
+                        }
+                }
+        }
+
+        /* Ironlake: try to setup display ref clock before DPLL
+         * enabling. This is only under driver's control after
+         * PCH B stepping, previous chipset stepping should be
+         * ignoring this setting.
+         */
+        temp = I915_READ(PCH_DREF_CONTROL);
+        /* Always enable nonspread source */
+        temp &= ~DREF_NONSPREAD_SOURCE_MASK;
+        temp |= DREF_NONSPREAD_SOURCE_ENABLE;
+        temp &= ~DREF_SSC_SOURCE_MASK;
+        temp |= DREF_SSC_SOURCE_ENABLE;
+        I915_WRITE(PCH_DREF_CONTROL, temp);
+
+        POSTING_READ(PCH_DREF_CONTROL);
+        udelay(200);
+
+        if (has_edp_encoder) {
+                if (intel_panel_use_ssc(dev_priv)) {
+                        temp |= DREF_SSC1_ENABLE;
+                        I915_WRITE(PCH_DREF_CONTROL, temp);
+
+                        POSTING_READ(PCH_DREF_CONTROL);
+                        udelay(200);
+                }
+                temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
+
+                /* Enable CPU source on CPU attached eDP */
+                if (!intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+                        if (intel_panel_use_ssc(dev_priv))
+                                temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
+                        else
+                                temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
+                } else {
+                        /* Enable SSC on PCH eDP if needed */
+                        if (intel_panel_use_ssc(dev_priv)) {
+                                DRM_ERROR("enabling SSC on PCH\n");
+                                temp |= DREF_SUPERSPREAD_SOURCE_ENABLE;
+                        }
+                }
+                I915_WRITE(PCH_DREF_CONTROL, temp);
+                POSTING_READ(PCH_DREF_CONTROL);
+                udelay(200);
+        }
+}
+
 static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
                                   struct drm_display_mode *mode,
                                   struct drm_display_mode *adjusted_mode,
@@ -4721,7 +5208,9 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         struct fdi_m_n m_n = {0};
         u32 temp;
         u32 lvds_sync = 0;
-        int target_clock, pixel_multiplier, lane, link_bw, bpp, factor;
+        int target_clock, pixel_multiplier, lane, link_bw, factor;
+        unsigned int pipe_bpp;
+        bool dither;
 
         list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
                 if (encoder->base.crtc != crtc)
@@ -4848,56 +5337,38 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         /* determine panel color depth */
         temp = I915_READ(PIPECONF(pipe));
         temp &= ~PIPE_BPC_MASK;
-        if (is_lvds) {
-                /* the BPC will be 6 if it is 18-bit LVDS panel */
-                if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) == LVDS_A3_POWER_UP)
-                        temp |= PIPE_8BPC;
-                else
-                        temp |= PIPE_6BPC;
-        } else if (has_edp_encoder) {
-                switch (dev_priv->edp.bpp/3) {
-                case 8:
-                        temp |= PIPE_8BPC;
-                        break;
-                case 10:
-                        temp |= PIPE_10BPC;
-                        break;
-                case 6:
-                        temp |= PIPE_6BPC;
-                        break;
-                case 12:
-                        temp |= PIPE_12BPC;
-                        break;
-                }
-        } else
-                temp |= PIPE_8BPC;
-        I915_WRITE(PIPECONF(pipe), temp);
-
-        switch (temp & PIPE_BPC_MASK) {
-        case PIPE_8BPC:
-                bpp = 24;
+        dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp);
+        switch (pipe_bpp) {
+        case 18:
+                temp |= PIPE_6BPC;
                 break;
-        case PIPE_10BPC:
-                bpp = 30;
+        case 24:
+                temp |= PIPE_8BPC;
                 break;
-        case PIPE_6BPC:
-                bpp = 18;
+        case 30:
+                temp |= PIPE_10BPC;
                 break;
-        case PIPE_12BPC:
-                bpp = 36;
+        case 36:
+                temp |= PIPE_12BPC;
                 break;
         default:
-                DRM_ERROR("unknown pipe bpc value\n");
-                bpp = 24;
+                WARN(1, "intel_choose_pipe_bpp returned invalid value %d\n",
+                        pipe_bpp);
+                temp |= PIPE_8BPC;
+                pipe_bpp = 24;
+                break;
         }
 
+        intel_crtc->bpp = pipe_bpp;
+        I915_WRITE(PIPECONF(pipe), temp);
+
         if (!lane) {
                 /*
                  * Account for spread spectrum to avoid
                  * oversubscribing the link. Max center spread
                  * is 2.5%; use 5% for safety's sake.
                  */
-                u32 bps = target_clock * bpp * 21 / 20;
+                u32 bps = target_clock * intel_crtc->bpp * 21 / 20;
                 lane = bps / (link_bw * 8) + 1;
         }
 
@@ -4905,51 +5376,10 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 
         if (pixel_multiplier > 1)
                 link_bw *= pixel_multiplier;
-        ironlake_compute_m_n(bpp, lane, target_clock, link_bw, &m_n);
-
-        /* Ironlake: try to setup display ref clock before DPLL
-         * enabling. This is only under driver's control after
-         * PCH B stepping, previous chipset stepping should be
-         * ignoring this setting.
-         */
-        temp = I915_READ(PCH_DREF_CONTROL);
-        /* Always enable nonspread source */
-        temp &= ~DREF_NONSPREAD_SOURCE_MASK;
-        temp |= DREF_NONSPREAD_SOURCE_ENABLE;
-        temp &= ~DREF_SSC_SOURCE_MASK;
-        temp |= DREF_SSC_SOURCE_ENABLE;
-        I915_WRITE(PCH_DREF_CONTROL, temp);
-
-        POSTING_READ(PCH_DREF_CONTROL);
-        udelay(200);
+        ironlake_compute_m_n(intel_crtc->bpp, lane, target_clock, link_bw,
+                             &m_n);
 
-        if (has_edp_encoder) {
-                if (intel_panel_use_ssc(dev_priv)) {
-                        temp |= DREF_SSC1_ENABLE;
-                        I915_WRITE(PCH_DREF_CONTROL, temp);
-
-                        POSTING_READ(PCH_DREF_CONTROL);
-                        udelay(200);
-                }
-                temp &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
-
-                /* Enable CPU source on CPU attached eDP */
-                if (!intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
-                        if (intel_panel_use_ssc(dev_priv))
-                                temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
-                        else
-                                temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
-                } else {
-                        /* Enable SSC on PCH eDP if needed */
-                        if (intel_panel_use_ssc(dev_priv)) {
-                                DRM_ERROR("enabling SSC on PCH\n");
-                                temp |= DREF_SUPERSPREAD_SOURCE_ENABLE;
-                        }
-                }
-                I915_WRITE(PCH_DREF_CONTROL, temp);
-                POSTING_READ(PCH_DREF_CONTROL);
-                udelay(200);
-        }
+        ironlake_update_pch_refclk(dev);
 
         fp = clock.n << 16 | clock.m1 << 8 | clock.m2;
         if (has_reduced_clock)
@@ -4966,7 +5396,7 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         } else if (is_sdvo && is_tv)
                 factor = 20;
 
-        if (clock.m1 < factor * clock.n)
+        if (clock.m < factor * clock.n)
                 fp |= FP_CB_TUNE;
 
         dpll = 0;
@@ -5108,14 +5538,12 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
                 I915_WRITE(PCH_LVDS, temp);
         }
 
-        /* set the dithering flag and clear for anything other than a panel. */
         pipeconf &= ~PIPECONF_DITHER_EN;
         pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
-        if (dev_priv->lvds_dither && (is_lvds || has_edp_encoder)) {
+        if ((is_lvds && dev_priv->lvds_dither) || dither) {
                 pipeconf |= PIPECONF_DITHER_EN;
                 pipeconf |= PIPECONF_DITHER_TYPE_ST1;
         }
-
         if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
                 intel_dp_set_m_n(crtc, mode, adjusted_mode);
         } else {
@@ -5246,6 +5674,8 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
 
         drm_vblank_post_modeset(dev, pipe);
 
+        intel_crtc->dpms_mode = DRM_MODE_DPMS_ON;
+
         return ret;
 }
 
@@ -5330,6 +5760,31 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
         I915_WRITE(CURBASE(pipe), base);
 }
 
+static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pipe = intel_crtc->pipe;
+        bool visible = base != 0;
+
+        if (intel_crtc->cursor_visible != visible) {
+                uint32_t cntl = I915_READ(CURCNTR_IVB(pipe));
+                if (base) {
+                        cntl &= ~CURSOR_MODE;
+                        cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
+                } else {
+                        cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
+                        cntl |= CURSOR_MODE_DISABLE;
+                }
+                I915_WRITE(CURCNTR_IVB(pipe), cntl);
+
+                intel_crtc->cursor_visible = visible;
+        }
+        /* and commit changes on next vblank */
+        I915_WRITE(CURBASE_IVB(pipe), base);
+}
+
 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
                                      bool on)
@@ -5377,11 +5832,16 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
         if (!visible && !intel_crtc->cursor_visible)
                 return;
 
-        I915_WRITE(CURPOS(pipe), pos);
-        if (IS_845G(dev) || IS_I865G(dev))
-                i845_update_cursor(crtc, base);
-        else
-                i9xx_update_cursor(crtc, base);
+        if (IS_IVYBRIDGE(dev)) {
+                I915_WRITE(CURPOS_IVB(pipe), pos);
+                ivb_update_cursor(crtc, base);
+        } else {
+                I915_WRITE(CURPOS(pipe), pos);
+                if (IS_845G(dev) || IS_I865G(dev))
+                        i845_update_cursor(crtc, base);
+                else
+                        i9xx_update_cursor(crtc, base);
+        }
 
         if (visible)
                 intel_mark_busy(dev, to_intel_framebuffer(crtc->fb)->obj);
@@ -5435,21 +5895,15 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                         goto fail_locked;
                 }
 
-                ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
-                if (ret) {
-                        DRM_ERROR("failed to pin cursor bo\n");
-                        goto fail_locked;
-                }
-
-                ret = i915_gem_object_set_to_gtt_domain(obj, 0);
+                ret = i915_gem_object_pin_to_display_plane(obj, 0, NULL);
                 if (ret) {
                         DRM_ERROR("failed to move cursor bo into the GTT\n");
-                        goto fail_unpin;
+                        goto fail_locked;
                 }
 
                 ret = i915_gem_object_put_fence(obj);
                 if (ret) {
-                        DRM_ERROR("failed to move cursor bo into the GTT\n");
+                        DRM_ERROR("failed to release fence for cursor");
                         goto fail_unpin;
                 }
 
@@ -6152,6 +6606,7 @@ static void intel_unpin_work_fn(struct work_struct *__work)
         drm_gem_object_unreference(&work->pending_flip_obj->base);
         drm_gem_object_unreference(&work->old_fb_obj->base);
 
+        intel_update_fbc(work->dev);
         mutex_unlock(&work->dev->struct_mutex);
         kfree(work);
 }
@@ -6516,6 +6971,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         if (ret)
                 goto cleanup_pending;
 
+        intel_disable_fbc(dev);
         mutex_unlock(&dev->struct_mutex);
 
         trace_i915_flip_request(intel_crtc->plane, obj);
@@ -6644,6 +7100,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 
         intel_crtc_reset(&intel_crtc->base);
         intel_crtc->active = true; /* force the pipe off on setup_init_config */
+        intel_crtc->bpp = 24; /* default for pre-Ironlake */
 
         if (HAS_PCH_SPLIT(dev)) {
                 intel_helper_funcs.prepare = ironlake_crtc_prepare;
@@ -6823,8 +7280,6 @@ static void intel_setup_outputs(struct drm_device *dev)
                         intel_encoder_clones(dev, encoder->clone_mask);
         }
 
-        intel_panel_setup_backlight(dev);
-
         /* disable all the possible outputs/crtcs before entering KMS mode */
         drm_helper_disable_unused_functions(dev);
 }
@@ -6870,6 +7325,11 @@ int intel_framebuffer_init(struct drm_device *dev,
         switch (mode_cmd->bpp) {
         case 8:
         case 16:
+                /* Only pre-ILK can handle 5:5:5 */
+                if (mode_cmd->depth == 15 && !HAS_PCH_SPLIT(dev))
+                        return -EINVAL;
+                break;
+
         case 24:
         case 32:
                 break;
@@ -7284,6 +7744,59 @@ void gen6_enable_rps(struct drm_i915_private *dev_priv)
         mutex_unlock(&dev_priv->dev->struct_mutex);
 }
 
+void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
+{
+        int min_freq = 15;
+        int gpu_freq, ia_freq, max_ia_freq;
+        int scaling_factor = 180;
+
+        max_ia_freq = cpufreq_quick_get_max(0);
+        /*
+         * Default to measured freq if none found, PCU will ensure we don't go
+         * over
+         */
+        if (!max_ia_freq)
+                max_ia_freq = tsc_khz;
+
+        /* Convert from kHz to MHz */
+        max_ia_freq /= 1000;
+
+        mutex_lock(&dev_priv->dev->struct_mutex);
+
+        /*
+         * For each potential GPU frequency, load a ring frequency we'd like
+         * to use for memory access.  We do this by specifying the IA frequency
+         * the PCU should use as a reference to determine the ring frequency.
+         */
+        for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
+             gpu_freq--) {
+                int diff = dev_priv->max_delay - gpu_freq;
+
+                /*
+                 * For GPU frequencies less than 750MHz, just use the lowest
+                 * ring freq.
+                 */
+                if (gpu_freq < min_freq)
+                        ia_freq = 800;
+                else
+                        ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
+                ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
+
+                I915_WRITE(GEN6_PCODE_DATA,
+                           (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
+                           gpu_freq);
+                I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
+                           GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
+                if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
+                              GEN6_PCODE_READY) == 0, 10)) {
+                        DRM_ERROR("pcode write of freq table timed out\n");
+                        continue;
+                }
+        }
+
+        mutex_unlock(&dev_priv->dev->struct_mutex);
+}
+
 static void ironlake_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -7369,6 +7882,20 @@ static void gen6_init_clock_gating(struct drm_device *dev)
         I915_WRITE(WM2_LP_ILK, 0);
         I915_WRITE(WM1_LP_ILK, 0);
 
+        /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
+         * gating disable must be set.  Failure to set it results in
+         * flickering pixels due to Z write ordering failures after
+         * some amount of runtime in the Mesa "fire" demo, and Unigine
+         * Sanctuary and Tropics, and apparently anything else with
+         * alpha test or pixel discard.
+         *
+         * According to the spec, bit 11 (RCCUNIT) must also be set,
+         * but we didn't debug actual testcases to find it out.
+         */
+        I915_WRITE(GEN6_UCGCTL2,
+                   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
+                   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
+
         /*
          * According to the spec the following bits should be
          * set in order to enable memory self-refresh and fbc:
@@ -7389,10 +7916,12 @@ static void gen6_init_clock_gating(struct drm_device *dev)
                    ILK_DPARB_CLK_GATE  |
                    ILK_DPFD_CLK_GATE);
 
-        for_each_pipe(pipe)
+        for_each_pipe(pipe) {
                 I915_WRITE(DSPCNTR(pipe),
                            I915_READ(DSPCNTR(pipe)) |
                            DISPPLANE_TRICKLE_FEED_DISABLE);
+                intel_flush_display_plane(dev_priv, pipe);
+        }
 }
 
 static void ivybridge_init_clock_gating(struct drm_device *dev)
@@ -7409,10 +7938,12 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 
         I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
 
-        for_each_pipe(pipe)
+        for_each_pipe(pipe) {
                 I915_WRITE(DSPCNTR(pipe),
                            I915_READ(DSPCNTR(pipe)) |
                            DISPPLANE_TRICKLE_FEED_DISABLE);
+                intel_flush_display_plane(dev_priv, pipe);
+        }
 }
 
 static void g4x_init_clock_gating(struct drm_device *dev)
@@ -7495,6 +8026,7 @@ static void ibx_init_clock_gating(struct drm_device *dev)
 static void cpt_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
 
         /*
          * On Ibex Peak and Cougar Point, we need to disable clock
@@ -7504,6 +8036,9 @@ static void cpt_init_clock_gating(struct drm_device *dev)
         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
                    DPLS_EDP_PPS_FIX_DIS);
+        /* Without this, mode sets may fail silently on FDI */
+        for_each_pipe(pipe)
+                I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
 }
 
 static void ironlake_teardown_rc6(struct drm_device *dev)
@@ -7640,9 +8175,11 @@ static void intel_init_display(struct drm_device *dev)
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->display.dpms = ironlake_crtc_dpms;
                 dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
+                dev_priv->display.update_plane = ironlake_update_plane;
         } else {
                 dev_priv->display.dpms = i9xx_crtc_dpms;
                 dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
+                dev_priv->display.update_plane = i9xx_update_plane;
         }
 
         if (I915_HAS_FBC(dev)) {
@@ -7851,6 +8388,9 @@ struct intel_quirk intel_quirks[] = {
 
         /* Lenovo U160 cannot use SSC on LVDS */
         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
+
+        /* Sony Vaio Y cannot use SSC on LVDS */
+        { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -7939,8 +8479,10 @@ void intel_modeset_init(struct drm_device *dev)
                 intel_init_emon(dev);
         }
 
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev) || IS_GEN7(dev)) {
                 gen6_enable_rps(dev_priv);
+                gen6_update_ring_freq(dev_priv);
+        }
 
         INIT_WORK(&dev_priv->idle_work, intel_idle_update);
         setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
@@ -7976,12 +8518,11 @@ void intel_modeset_cleanup(struct drm_device *dev)
                 intel_increase_pllclock(crtc);
         }
 
-        if (dev_priv->display.disable_fbc)
-                dev_priv->display.disable_fbc(dev);
+        intel_disable_fbc(dev);
 
         if (IS_IRONLAKE_M(dev))
                 ironlake_disable_drps(dev);
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev) || IS_GEN7(dev))
                 gen6_disable_rps(dev);
 
         if (IS_IRONLAKE_M(dev))
@@ -7994,6 +8535,9 @@ void intel_modeset_cleanup(struct drm_device *dev)
         drm_irq_uninstall(dev);
         cancel_work_sync(&dev_priv->hotplug_work);
 
+        /* flush any delayed tasks or pending work */
+        flush_scheduled_work();
+
         /* Shut off idle work before the crtcs get freed. */
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 intel_crtc = to_intel_crtc(crtc);
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index e2aced6eec4..7c64db48c1c 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -50,9 +50,10 @@ struct intel_dp {
         bool has_audio;
         int force_audio;
         uint32_t color_range;
+        int dpms_mode;
         uint8_t link_bw;
         uint8_t lane_count;
-        uint8_t dpcd[4];
+        uint8_t dpcd[8];
         struct i2c_adapter adapter;
         struct i2c_algo_dp_aux_data algo;
         bool is_pch_edp;
@@ -178,12 +179,14 @@ intel_dp_link_clock(uint8_t link_bw)
 static int
 intel_dp_link_required(struct drm_device *dev, struct intel_dp *intel_dp, int pixel_clock)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = intel_dp->base.base.crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int bpp = 24;
 
-        if (is_edp(intel_dp))
-                return (pixel_clock * dev_priv->edp.bpp + 7) / 8;
-        else
-                return pixel_clock * 3;
+        if (intel_crtc)
+                bpp = intel_crtc->bpp;
+
+        return (pixel_clock * bpp + 7) / 8;
 }
 
 static int
@@ -314,9 +317,17 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
         else
                 precharge = 5;
 
-        if (I915_READ(ch_ctl) & DP_AUX_CH_CTL_SEND_BUSY) {
-                DRM_ERROR("dp_aux_ch not started status 0x%08x\n",
-                          I915_READ(ch_ctl));
+        /* Try to wait for any previous AUX channel activity */
+        for (try = 0; try < 3; try++) {
+                status = I915_READ(ch_ctl);
+                if ((status & DP_AUX_CH_CTL_SEND_BUSY) == 0)
+                        break;
+                msleep(1);
+        }
+
+        if (try == 3) {
+                WARN(1, "dp_aux_ch not started status 0x%08x\n",
+                     I915_READ(ch_ctl));
                 return -EBUSY;
         }
 
@@ -681,7 +692,7 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
         struct drm_encoder *encoder;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int lane_count = 4, bpp = 24;
+        int lane_count = 4;
         struct intel_dp_m_n m_n;
         int pipe = intel_crtc->pipe;
 
@@ -700,7 +711,6 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
                         break;
                 } else if (is_edp(intel_dp)) {
                         lane_count = dev_priv->edp.lanes;
-                        bpp = dev_priv->edp.bpp;
                         break;
                 }
         }
@@ -710,7 +720,7 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
          * the number of bytes_per_pixel post-LUT, which we always
          * set up for 8-bits of R/G/B, or 3 bytes total.
          */
-        intel_dp_compute_m_n(bpp, lane_count,
+        intel_dp_compute_m_n(intel_crtc->bpp, lane_count,
                              mode->clock, adjusted_mode->clock, &m_n);
 
         if (HAS_PCH_SPLIT(dev)) {
@@ -769,6 +779,7 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
         memset(intel_dp->link_configuration, 0, DP_LINK_CONFIGURATION_SIZE);
         intel_dp->link_configuration[0] = intel_dp->link_bw;
         intel_dp->link_configuration[1] = intel_dp->lane_count;
+        intel_dp->link_configuration[8] = DP_SET_ANSI_8B10B;
 
         /*
          * Check for DPCD version > 1.1 and enhanced framing support
@@ -1010,6 +1021,8 @@ static void intel_dp_commit(struct drm_encoder *encoder)
 
         if (is_edp(intel_dp))
                 ironlake_edp_backlight_on(dev);
+
+        intel_dp->dpms_mode = DRM_MODE_DPMS_ON;
 }
 
 static void
@@ -1044,6 +1057,7 @@ intel_dp_dpms(struct drm_encoder *encoder, int mode)
                 if (is_edp(intel_dp))
                         ironlake_edp_backlight_on(dev);
         }
+        intel_dp->dpms_mode = mode;
 }
 
 /*
@@ -1333,10 +1347,16 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
         u32 reg;
         uint32_t DP = intel_dp->DP;
 
-        /* Enable output, wait for it to become active */
-        I915_WRITE(intel_dp->output_reg, intel_dp->DP);
-        POSTING_READ(intel_dp->output_reg);
-        intel_wait_for_vblank(dev, intel_crtc->pipe);
+        /*
+         * On CPT we have to enable the port in training pattern 1, which
+         * will happen below in intel_dp_set_link_train.  Otherwise, enable
+         * the port and wait for it to become active.
+         */
+        if (!HAS_PCH_CPT(dev)) {
+                I915_WRITE(intel_dp->output_reg, intel_dp->DP);
+                POSTING_READ(intel_dp->output_reg);
+                intel_wait_for_vblank(dev, intel_crtc->pipe);
+        }
 
         /* Write the link configuration data */
         intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET,
@@ -1369,7 +1389,8 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
                         reg = DP | DP_LINK_TRAIN_PAT_1;
 
                 if (!intel_dp_set_link_train(intel_dp, reg,
-                                             DP_TRAINING_PATTERN_1))
+                                             DP_TRAINING_PATTERN_1 |
+                                             DP_LINK_SCRAMBLING_DISABLE))
                         break;
                 /* Set training pattern 1 */
 
@@ -1444,7 +1465,8 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
 
                 /* channel eq pattern */
                 if (!intel_dp_set_link_train(intel_dp, reg,
-                                             DP_TRAINING_PATTERN_2))
+                                             DP_TRAINING_PATTERN_2 |
+                                             DP_LINK_SCRAMBLING_DISABLE))
                         break;
 
                 udelay(400);
@@ -1558,6 +1580,18 @@ intel_dp_link_down(struct intel_dp *intel_dp)
         POSTING_READ(intel_dp->output_reg);
 }
 
+static bool
+intel_dp_get_dpcd(struct intel_dp *intel_dp)
+{
+        if (intel_dp_aux_native_read_retry(intel_dp, 0x000, intel_dp->dpcd,
+                                           sizeof (intel_dp->dpcd)) &&
+            (intel_dp->dpcd[DP_DPCD_REV] != 0)) {
+                return true;
+        }
+
+        return false;
+}
+
 /*
  * According to DP spec
  * 5.1.2:
@@ -1570,36 +1604,44 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 static void
 intel_dp_check_link_status(struct intel_dp *intel_dp)
 {
-        int ret;
+        if (intel_dp->dpms_mode != DRM_MODE_DPMS_ON)
+                return;
 
         if (!intel_dp->base.base.crtc)
                 return;
 
+        /* Try to read receiver status if the link appears to be up */
         if (!intel_dp_get_link_status(intel_dp)) {
                 intel_dp_link_down(intel_dp);
                 return;
         }
 
-        /* Try to read receiver status if the link appears to be up */
-        ret = intel_dp_aux_native_read(intel_dp,
-                                       0x000, intel_dp->dpcd,
-                                       sizeof (intel_dp->dpcd));
-        if (ret != sizeof(intel_dp->dpcd)) {
+        /* Now read the DPCD to see if it's actually running */
+        if (!intel_dp_get_dpcd(intel_dp)) {
                 intel_dp_link_down(intel_dp);
                 return;
         }
 
         if (!intel_channel_eq_ok(intel_dp)) {
+                DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n",
+                              drm_get_encoder_name(&intel_dp->base.base));
                 intel_dp_start_link_train(intel_dp);
                 intel_dp_complete_link_train(intel_dp);
         }
 }
 
 static enum drm_connector_status
+intel_dp_detect_dpcd(struct intel_dp *intel_dp)
+{
+        if (intel_dp_get_dpcd(intel_dp))
+                return connector_status_connected;
+        return connector_status_disconnected;
+}
+
+static enum drm_connector_status
 ironlake_dp_detect(struct intel_dp *intel_dp)
 {
         enum drm_connector_status status;
-        bool ret;
 
         /* Can't disconnect eDP, but you can close the lid... */
         if (is_edp(intel_dp)) {
@@ -1609,15 +1651,7 @@ ironlake_dp_detect(struct intel_dp *intel_dp)
                 return status;
         }
 
-        status = connector_status_disconnected;
-        ret = intel_dp_aux_native_read_retry(intel_dp,
-                                             0x000, intel_dp->dpcd,
-                                             sizeof (intel_dp->dpcd));
-        if (ret && intel_dp->dpcd[DP_DPCD_REV] != 0)
-                status = connector_status_connected;
-        DRM_DEBUG_KMS("DPCD: %hx%hx%hx%hx\n", intel_dp->dpcd[0],
-                      intel_dp->dpcd[1], intel_dp->dpcd[2], intel_dp->dpcd[3]);
-        return status;
+        return intel_dp_detect_dpcd(intel_dp);
 }
 
 static enum drm_connector_status
@@ -1625,7 +1659,6 @@ g4x_dp_detect(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum drm_connector_status status;
         uint32_t temp, bit;
 
         switch (intel_dp->output_reg) {
@@ -1647,17 +1680,34 @@ g4x_dp_detect(struct intel_dp *intel_dp)
         if ((temp & bit) == 0)
                 return connector_status_disconnected;
 
-        status = connector_status_disconnected;
-        if (intel_dp_aux_native_read(intel_dp, 0x000, intel_dp->dpcd,
-                                     sizeof (intel_dp->dpcd)) == sizeof (intel_dp->dpcd))
-        {
-                if (intel_dp->dpcd[DP_DPCD_REV] != 0)
-                        status = connector_status_connected;
-        }
+        return intel_dp_detect_dpcd(intel_dp);
+}
+
+static struct edid *
+intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
+{
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        struct edid     *edid;
 
-        return status;
+        ironlake_edp_panel_vdd_on(intel_dp);
+        edid = drm_get_edid(connector, adapter);
+        ironlake_edp_panel_vdd_off(intel_dp);
+        return edid;
 }
 
+static int
+intel_dp_get_edid_modes(struct drm_connector *connector, struct i2c_adapter *adapter)
+{
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        int     ret;
+
+        ironlake_edp_panel_vdd_on(intel_dp);
+        ret = intel_ddc_get_modes(connector, adapter);
+        ironlake_edp_panel_vdd_off(intel_dp);
+        return ret;
+}
+
+
 /**
  * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection.
  *
@@ -1678,13 +1728,19 @@ intel_dp_detect(struct drm_connector *connector, bool force)
                 status = ironlake_dp_detect(intel_dp);
         else
                 status = g4x_dp_detect(intel_dp);
+
+        DRM_DEBUG_KMS("DPCD: %02hx%02hx%02hx%02hx%02hx%02hx%02hx%02hx\n",
+                      intel_dp->dpcd[0], intel_dp->dpcd[1], intel_dp->dpcd[2],
+                      intel_dp->dpcd[3], intel_dp->dpcd[4], intel_dp->dpcd[5],
+                      intel_dp->dpcd[6], intel_dp->dpcd[7]);
+
         if (status != connector_status_connected)
                 return status;
 
         if (intel_dp->force_audio) {
                 intel_dp->has_audio = intel_dp->force_audio > 0;
         } else {
-                edid = drm_get_edid(connector, &intel_dp->adapter);
+                edid = intel_dp_get_edid(connector, &intel_dp->adapter);
                 if (edid) {
                         intel_dp->has_audio = drm_detect_monitor_audio(edid);
                         connector->display_info.raw_edid = NULL;
@@ -1705,7 +1761,7 @@ static int intel_dp_get_modes(struct drm_connector *connector)
         /* We should parse the EDID data and find out if it has an audio sink
          */
 
-        ret = intel_ddc_get_modes(connector, &intel_dp->adapter);
+        ret = intel_dp_get_edid_modes(connector, &intel_dp->adapter);
         if (ret) {
                 if (is_edp(intel_dp) && !dev_priv->panel_fixed_mode) {
                         struct drm_display_mode *newmode;
@@ -1741,7 +1797,7 @@ intel_dp_detect_audio(struct drm_connector *connector)
         struct edid *edid;
         bool has_audio = false;
 
-        edid = drm_get_edid(connector, &intel_dp->adapter);
+        edid = intel_dp_get_edid(connector, &intel_dp->adapter);
         if (edid) {
                 has_audio = drm_detect_monitor_audio(edid);
 
@@ -1810,6 +1866,11 @@ done:
 static void
 intel_dp_destroy (struct drm_connector *connector)
 {
+        struct drm_device *dev = connector->dev;
+
+        if (intel_dpd_is_edp(dev))
+                intel_panel_destroy_backlight(dev);
+
         drm_sysfs_connector_remove(connector);
         drm_connector_cleanup(connector);
         kfree(connector);
@@ -1923,6 +1984,7 @@ intel_dp_init(struct drm_device *dev, int output_reg)
                 return;
 
         intel_dp->output_reg = output_reg;
+        intel_dp->dpms_mode = -1;
 
         intel_connector = kzalloc(sizeof(struct intel_connector), GFP_KERNEL);
         if (!intel_connector) {
@@ -1999,7 +2061,7 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 
         /* Cache some DPCD data in the eDP case */
         if (is_edp(intel_dp)) {
-                int ret;
+                bool ret;
                 u32 pp_on, pp_div;
 
                 pp_on = I915_READ(PCH_PP_ON_DELAYS);
@@ -2012,11 +2074,9 @@ intel_dp_init(struct drm_device *dev, int output_reg)
                 dev_priv->panel_t12 *= 100; /* t12 in 100ms units */
 
                 ironlake_edp_panel_vdd_on(intel_dp);
-                ret = intel_dp_aux_native_read(intel_dp, DP_DPCD_REV,
-                                               intel_dp->dpcd,
-                                               sizeof(intel_dp->dpcd));
+                ret = intel_dp_get_dpcd(intel_dp);
                 ironlake_edp_panel_vdd_off(intel_dp);
-                if (ret == sizeof(intel_dp->dpcd)) {
+                if (ret) {
                         if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
                                 dev_priv->no_aux_handshake =
                                         intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
@@ -2042,6 +2102,8 @@ intel_dp_init(struct drm_device *dev, int output_reg)
                                         DRM_MODE_TYPE_PREFERRED;
                         }
                 }
+                dev_priv->int_edp_connector = connector;
+                intel_panel_setup_backlight(dev);
         }
 
         intel_dp_add_properties(intel_dp, connector);
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 9ffa61eb4d7..2480cfa7c0c 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -170,6 +170,7 @@ struct intel_crtc {
         int16_t cursor_x, cursor_y;
         int16_t cursor_width, cursor_height;
         bool cursor_visible;
+        unsigned int bpp;
 };
 
 #define to_intel_crtc(x) container_of(x, struct intel_crtc, base)
@@ -177,10 +178,28 @@ struct intel_crtc {
 #define to_intel_encoder(x) container_of(x, struct intel_encoder, base)
 #define to_intel_framebuffer(x) container_of(x, struct intel_framebuffer, base)
 
+#define DIP_HEADER_SIZE 5
+
 #define DIP_TYPE_AVI    0x82
 #define DIP_VERSION_AVI 0x2
 #define DIP_LEN_AVI     13
 
+#define DIP_TYPE_SPD    0x83
+#define DIP_VERSION_SPD 0x1
+#define DIP_LEN_SPD     25
+#define DIP_SPD_UNKNOWN 0
+#define DIP_SPD_DSTB    0x1
+#define DIP_SPD_DVDP    0x2
+#define DIP_SPD_DVHS    0x3
+#define DIP_SPD_HDDVR   0x4
+#define DIP_SPD_DVC     0x5
+#define DIP_SPD_DSC     0x6
+#define DIP_SPD_VCD     0x7
+#define DIP_SPD_GAME    0x8
+#define DIP_SPD_PC      0x9
+#define DIP_SPD_BD      0xa
+#define DIP_SPD_SCD     0xb
+
 struct dip_infoframe {
         uint8_t type;           /* HB0 */
         uint8_t ver;            /* HB1 */
@@ -205,6 +224,11 @@ struct dip_infoframe {
                         uint16_t left_bar_end;
                         uint16_t right_bar_start;
                 } avi;
+                struct {
+                        uint8_t vn[8];
+                        uint8_t pd[16];
+                        uint8_t sdi;
+                } spd;
                 uint8_t payload[27];
         } __attribute__ ((packed)) body;
 } __attribute__((packed));
@@ -233,6 +257,13 @@ struct intel_unpin_work {
         bool enable_stall_check;
 };
 
+struct intel_fbc_work {
+        struct delayed_work work;
+        struct drm_crtc *crtc;
+        struct drm_framebuffer *fb;
+        int interval;
+};
+
 int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
 extern bool intel_ddc_probe(struct intel_encoder *intel_encoder, int ddc_bus);
 
@@ -266,9 +297,10 @@ extern void intel_pch_panel_fitting(struct drm_device *dev,
 extern u32 intel_panel_get_max_backlight(struct drm_device *dev);
 extern u32 intel_panel_get_backlight(struct drm_device *dev);
 extern void intel_panel_set_backlight(struct drm_device *dev, u32 level);
-extern void intel_panel_setup_backlight(struct drm_device *dev);
+extern int intel_panel_setup_backlight(struct drm_device *dev);
 extern void intel_panel_enable_backlight(struct drm_device *dev);
 extern void intel_panel_disable_backlight(struct drm_device *dev);
+extern void intel_panel_destroy_backlight(struct drm_device *dev);
 extern enum drm_connector_status intel_panel_detect(struct drm_device *dev);
 
 extern void intel_crtc_load_lut(struct drm_crtc *crtc);
@@ -305,9 +337,6 @@ extern void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
                                            struct drm_connector *connector,
                                            struct intel_load_detect_pipe *old);
 
-extern struct drm_connector* intel_sdvo_find(struct drm_device *dev, int sdvoB);
-extern int intel_sdvo_supports_hotplug(struct drm_connector *connector);
-extern void intel_sdvo_set_hotplug(struct drm_connector *connector, int enable);
 extern void intelfb_restore(void);
 extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
                                     u16 blue, int regno);
@@ -317,6 +346,7 @@ extern void intel_enable_clock_gating(struct drm_device *dev);
 extern void ironlake_enable_drps(struct drm_device *dev);
 extern void ironlake_disable_drps(struct drm_device *dev);
 extern void gen6_enable_rps(struct drm_i915_private *dev_priv);
+extern void gen6_update_ring_freq(struct drm_i915_private *dev_priv);
 extern void gen6_disable_rps(struct drm_device *dev);
 extern void intel_init_emon(struct drm_device *dev);
 
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index aa0a8e83142..226ba830f38 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -45,6 +45,8 @@ struct intel_hdmi {
         bool has_hdmi_sink;
         bool has_audio;
         int force_audio;
+        void (*write_infoframe)(struct drm_encoder *encoder,
+                                struct dip_infoframe *frame);
 };
 
 static struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder)
@@ -58,37 +60,70 @@ static struct intel_hdmi *intel_attached_hdmi(struct drm_connector *connector)
                             struct intel_hdmi, base);
 }
 
-void intel_dip_infoframe_csum(struct dip_infoframe *avi_if)
+void intel_dip_infoframe_csum(struct dip_infoframe *frame)
 {
-        uint8_t *data = (uint8_t *)avi_if;
+        uint8_t *data = (uint8_t *)frame;
         uint8_t sum = 0;
         unsigned i;
 
-        avi_if->checksum = 0;
-        avi_if->ecc = 0;
+        frame->checksum = 0;
+        frame->ecc = 0;
 
-        for (i = 0; i < sizeof(*avi_if); i++)
+        /* Header isn't part of the checksum */
+        for (i = 5; i < frame->len; i++)
                 sum += data[i];
 
-        avi_if->checksum = 0x100 - sum;
+        frame->checksum = 0x100 - sum;
 }
 
-static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
+static u32 intel_infoframe_index(struct dip_infoframe *frame)
 {
-        struct dip_infoframe avi_if = {
-                .type = DIP_TYPE_AVI,
-                .ver = DIP_VERSION_AVI,
-                .len = DIP_LEN_AVI,
-        };
-        uint32_t *data = (uint32_t *)&avi_if;
+        u32 flags = 0;
+
+        switch (frame->type) {
+        case DIP_TYPE_AVI:
+                flags |= VIDEO_DIP_SELECT_AVI;
+                break;
+        case DIP_TYPE_SPD:
+                flags |= VIDEO_DIP_SELECT_SPD;
+                break;
+        default:
+                DRM_DEBUG_DRIVER("unknown info frame type %d\n", frame->type);
+                break;
+        }
+
+        return flags;
+}
+
+static u32 intel_infoframe_flags(struct dip_infoframe *frame)
+{
+        u32 flags = 0;
+
+        switch (frame->type) {
+        case DIP_TYPE_AVI:
+                flags |= VIDEO_DIP_ENABLE_AVI | VIDEO_DIP_FREQ_VSYNC;
+                break;
+        case DIP_TYPE_SPD:
+                flags |= VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_FREQ_2VSYNC;
+                break;
+        default:
+                DRM_DEBUG_DRIVER("unknown info frame type %d\n", frame->type);
+                break;
+        }
+
+        return flags;
+}
+
+static void i9xx_write_infoframe(struct drm_encoder *encoder,
+                                 struct dip_infoframe *frame)
+{
+        uint32_t *data = (uint32_t *)frame;
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-        u32 port;
-        unsigned i;
+        u32 port, flags, val = I915_READ(VIDEO_DIP_CTL);
+        unsigned i, len = DIP_HEADER_SIZE + frame->len;
 
-        if (!intel_hdmi->has_hdmi_sink)
-                return;
 
         /* XXX first guess at handling video port, is this corrent? */
         if (intel_hdmi->sdvox_reg == SDVOB)
@@ -98,18 +133,87 @@ static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
         else
                 return;
 
-        I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | port |
-                   VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC);
+        flags = intel_infoframe_index(frame);
+
+        val &= ~VIDEO_DIP_SELECT_MASK;
 
-        intel_dip_infoframe_csum(&avi_if);
-        for (i = 0; i < sizeof(avi_if); i += 4) {
+        I915_WRITE(VIDEO_DIP_CTL, val | port | flags);
+
+        for (i = 0; i < len; i += 4) {
                 I915_WRITE(VIDEO_DIP_DATA, *data);
                 data++;
         }
 
-        I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | port |
-                   VIDEO_DIP_SELECT_AVI | VIDEO_DIP_FREQ_VSYNC |
-                   VIDEO_DIP_ENABLE_AVI);
+        flags |= intel_infoframe_flags(frame);
+
+        I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | val | port | flags);
+}
+
+static void ironlake_write_infoframe(struct drm_encoder *encoder,
+                                     struct dip_infoframe *frame)
+{
+        uint32_t *data = (uint32_t *)frame;
+        struct drm_device *dev = encoder->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = encoder->crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+        unsigned i, len = DIP_HEADER_SIZE + frame->len;
+        u32 flags, val = I915_READ(reg);
+
+        intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+        flags = intel_infoframe_index(frame);
+
+        val &= ~VIDEO_DIP_SELECT_MASK;
+
+        I915_WRITE(reg, val | flags);
+
+        for (i = 0; i < len; i += 4) {
+                I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
+                data++;
+        }
+
+        flags |= intel_infoframe_flags(frame);
+
+        I915_WRITE(reg, VIDEO_DIP_ENABLE | val | flags);
+}
+static void intel_set_infoframe(struct drm_encoder *encoder,
+                                struct dip_infoframe *frame)
+{
+        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+
+        if (!intel_hdmi->has_hdmi_sink)
+                return;
+
+        intel_dip_infoframe_csum(frame);
+        intel_hdmi->write_infoframe(encoder, frame);
+}
+
+static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
+{
+        struct dip_infoframe avi_if = {
+                .type = DIP_TYPE_AVI,
+                .ver = DIP_VERSION_AVI,
+                .len = DIP_LEN_AVI,
+        };
+
+        intel_set_infoframe(encoder, &avi_if);
+}
+
+static void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder)
+{
+        struct dip_infoframe spd_if;
+
+        memset(&spd_if, 0, sizeof(spd_if));
+        spd_if.type = DIP_TYPE_SPD;
+        spd_if.ver = DIP_VERSION_SPD;
+        spd_if.len = DIP_LEN_SPD;
+        strcpy(spd_if.body.spd.vn, "Intel");
+        strcpy(spd_if.body.spd.pd, "Integrated gfx");
+        spd_if.body.spd.sdi = DIP_SPD_PC;
+
+        intel_set_infoframe(encoder, &spd_if);
 }
 
 static void intel_hdmi_mode_set(struct drm_encoder *encoder,
@@ -124,12 +228,18 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
         u32 sdvox;
 
         sdvox = SDVO_ENCODING_HDMI | SDVO_BORDER_ENABLE;
-        sdvox |= intel_hdmi->color_range;
+        if (!HAS_PCH_SPLIT(dev))
+                sdvox |= intel_hdmi->color_range;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                 sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
         if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                 sdvox |= SDVO_HSYNC_ACTIVE_HIGH;
 
+        if (intel_crtc->bpp > 24)
+                sdvox |= COLOR_FORMAT_12bpc;
+        else
+                sdvox |= COLOR_FORMAT_8bpc;
+
         /* Required on CPT */
         if (intel_hdmi->has_hdmi_sink && HAS_PCH_CPT(dev))
                 sdvox |= HDMI_MODE_SELECT;
@@ -150,6 +260,7 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
         POSTING_READ(intel_hdmi->sdvox_reg);
 
         intel_hdmi_set_avi_infoframe(encoder);
+        intel_hdmi_set_spd_infoframe(encoder);
 }
 
 static void intel_hdmi_dpms(struct drm_encoder *encoder, int mode)
@@ -427,6 +538,11 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
 
         intel_hdmi->sdvox_reg = sdvox_reg;
 
+        if (!HAS_PCH_SPLIT(dev))
+                intel_hdmi->write_infoframe = i9xx_write_infoframe;
+        else
+                intel_hdmi->write_infoframe = ironlake_write_infoframe;
+
         drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs);
 
         intel_hdmi_add_properties(intel_hdmi, connector);
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index b28f7bd9f88..31da77f5c05 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -72,14 +72,16 @@ static void intel_lvds_enable(struct intel_lvds *intel_lvds)
 {
         struct drm_device *dev = intel_lvds->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 ctl_reg, lvds_reg;
+        u32 ctl_reg, lvds_reg, stat_reg;
 
         if (HAS_PCH_SPLIT(dev)) {
                 ctl_reg = PCH_PP_CONTROL;
                 lvds_reg = PCH_LVDS;
+                stat_reg = PCH_PP_STATUS;
         } else {
                 ctl_reg = PP_CONTROL;
                 lvds_reg = LVDS;
+                stat_reg = PP_STATUS;
         }
 
         I915_WRITE(lvds_reg, I915_READ(lvds_reg) | LVDS_PORT_EN);
@@ -94,17 +96,16 @@ static void intel_lvds_enable(struct intel_lvds *intel_lvds)
                 DRM_DEBUG_KMS("applying panel-fitter: %x, %x\n",
                               intel_lvds->pfit_control,
                               intel_lvds->pfit_pgm_ratios);
-                if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000)) {
-                        DRM_ERROR("timed out waiting for panel to power off\n");
-                } else {
-                        I915_WRITE(PFIT_PGM_RATIOS, intel_lvds->pfit_pgm_ratios);
-                        I915_WRITE(PFIT_CONTROL, intel_lvds->pfit_control);
-                        intel_lvds->pfit_dirty = false;
-                }
+
+                I915_WRITE(PFIT_PGM_RATIOS, intel_lvds->pfit_pgm_ratios);
+                I915_WRITE(PFIT_CONTROL, intel_lvds->pfit_control);
+                intel_lvds->pfit_dirty = false;
         }
 
         I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON);
         POSTING_READ(lvds_reg);
+        if (wait_for((I915_READ(stat_reg) & PP_ON) != 0, 1000))
+                DRM_ERROR("timed out waiting for panel to power on\n");
 
         intel_panel_enable_backlight(dev);
 }
@@ -113,24 +114,25 @@ static void intel_lvds_disable(struct intel_lvds *intel_lvds)
 {
         struct drm_device *dev = intel_lvds->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 ctl_reg, lvds_reg;
+        u32 ctl_reg, lvds_reg, stat_reg;
 
         if (HAS_PCH_SPLIT(dev)) {
                 ctl_reg = PCH_PP_CONTROL;
                 lvds_reg = PCH_LVDS;
+                stat_reg = PCH_PP_STATUS;
         } else {
                 ctl_reg = PP_CONTROL;
                 lvds_reg = LVDS;
+                stat_reg = PP_STATUS;
         }
 
         intel_panel_disable_backlight(dev);
 
         I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
+        if (wait_for((I915_READ(stat_reg) & PP_ON) == 0, 1000))
+                DRM_ERROR("timed out waiting for panel to power off\n");
 
         if (intel_lvds->pfit_control) {
-                if (wait_for((I915_READ(PP_STATUS) & PP_ON) == 0, 1000))
-                        DRM_ERROR("timed out waiting for panel to power off\n");
-
                 I915_WRITE(PFIT_CONTROL, 0);
                 intel_lvds->pfit_dirty = true;
         }
@@ -398,53 +400,21 @@ out:
 
 static void intel_lvds_prepare(struct drm_encoder *encoder)
 {
-        struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
-        /* We try to do the minimum that is necessary in order to unlock
-         * the registers for mode setting.
-         *
-         * On Ironlake, this is quite simple as we just set the unlock key
-         * and ignore all subtleties. (This may cause some issues...)
-         *
+        /*
          * Prior to Ironlake, we must disable the pipe if we want to adjust
          * the panel fitter. However at all other times we can just reset
          * the registers regardless.
          */
-
-        if (HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(PCH_PP_CONTROL,
-                           I915_READ(PCH_PP_CONTROL) | PANEL_UNLOCK_REGS);
-        } else if (intel_lvds->pfit_dirty) {
-                I915_WRITE(PP_CONTROL,
-                           (I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS)
-                           & ~POWER_TARGET_ON);
-        } else {
-                I915_WRITE(PP_CONTROL,
-                           I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
-        }
+        if (!HAS_PCH_SPLIT(encoder->dev) && intel_lvds->pfit_dirty)
+                intel_lvds_disable(intel_lvds);
 }
 
 static void intel_lvds_commit(struct drm_encoder *encoder)
 {
-        struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
 
-        /* Undo any unlocking done in prepare to prevent accidental
-         * adjustment of the registers.
-         */
-        if (HAS_PCH_SPLIT(dev)) {
-                u32 val = I915_READ(PCH_PP_CONTROL);
-                if ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS)
-                        I915_WRITE(PCH_PP_CONTROL, val & 0x3);
-        } else {
-                u32 val = I915_READ(PP_CONTROL);
-                if ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS)
-                        I915_WRITE(PP_CONTROL, val & 0x3);
-        }
-
         /* Always do a full power on as we do not know what state
          * we were left in.
          */
@@ -582,6 +552,8 @@ static void intel_lvds_destroy(struct drm_connector *connector)
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        intel_panel_destroy_backlight(dev);
+
         if (dev_priv->lid_notifier.notifier_call)
                 acpi_lid_notifier_unregister(&dev_priv->lid_notifier);
         drm_sysfs_connector_remove(connector);
@@ -690,6 +662,14 @@ static const struct dmi_system_id intel_no_lvds[] = {
         },
         {
                 .callback = intel_no_lvds_dmi_callback,
+                .ident = "Dell OptiPlex FX170",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+                        DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex FX170"),
+                },
+        },
+        {
+                .callback = intel_no_lvds_dmi_callback,
                 .ident = "AOpen Mini PC",
                 .matches = {
                         DMI_MATCH(DMI_SYS_VENDOR, "AOpen"),
@@ -1032,6 +1012,19 @@ out:
                 pwm = I915_READ(BLC_PWM_PCH_CTL1);
                 pwm |= PWM_PCH_ENABLE;
                 I915_WRITE(BLC_PWM_PCH_CTL1, pwm);
+                /*
+                 * Unlock registers and just
+                 * leave them unlocked
+                 */
+                I915_WRITE(PCH_PP_CONTROL,
+                           I915_READ(PCH_PP_CONTROL) | PANEL_UNLOCK_REGS);
+        } else {
+                /*
+                 * Unlock registers and just
+                 * leave them unlocked
+                 */
+                I915_WRITE(PP_CONTROL,
+                           I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS);
         }
         dev_priv->lid_notifier.notifier_call = intel_lid_notify;
         if (acpi_lid_notifier_register(&dev_priv->lid_notifier)) {
@@ -1041,6 +1034,9 @@ out:
         /* keep the LVDS connector */
         dev_priv->int_lvds_connector = connector;
         drm_sysfs_connector_add(connector);
+
+        intel_panel_setup_backlight(dev);
+
         return true;
 
 failed:
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index d2c71042290..b8e8158bb16 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -227,7 +227,6 @@ void intel_opregion_asle_intr(struct drm_device *dev)
         asle->aslc = asle_stat;
 }
 
-/* Only present on Ironlake+ */
 void intel_opregion_gse_intr(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -297,19 +296,26 @@ static int intel_opregion_video_event(struct notifier_block *nb,
         /* The only video events relevant to opregion are 0x80. These indicate
            either a docking event, lid switch or display switch request. In
            Linux, these are handled by the dock, button and video drivers.
-           We might want to fix the video driver to be opregion-aware in
-           future, but right now we just indicate to the firmware that the
-           request has been handled */
+        */
 
         struct opregion_acpi *acpi;
+        struct acpi_bus_event *event = data;
+        int ret = NOTIFY_OK;
+
+        if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0)
+                return NOTIFY_DONE;
 
         if (!system_opregion)
                 return NOTIFY_DONE;
 
         acpi = system_opregion->acpi;
+
+        if (event->type == 0x80 && !(acpi->cevt & 0x1))
+                ret = NOTIFY_BAD;
+
         acpi->csts = 0;
 
-        return NOTIFY_OK;
+        return ret;
 }
 
 static struct notifier_block intel_opregion_notifier = {
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 9e2959bc91c..d3603808682 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -773,14 +773,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         if (ret != 0)
                 return ret;
 
-        ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true);
+        ret = i915_gem_object_pin_to_display_plane(new_bo, 0, NULL);
         if (ret != 0)
                 return ret;
 
-        ret = i915_gem_object_set_to_gtt_domain(new_bo, 0);
-        if (ret != 0)
-                goto out_unpin;
-
         ret = i915_gem_object_put_fence(new_bo);
         if (ret)
                 goto out_unpin;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index a06ff07a4d3..007f6ca309d 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -83,11 +83,15 @@ intel_pch_panel_fitting(struct drm_device *dev,
                         u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
                         if (scaled_width > scaled_height) { /* pillar */
                                 width = scaled_height / mode->vdisplay;
+                                if (width & 1)
+                                        width++;
                                 x = (adjusted_mode->hdisplay - width + 1) / 2;
                                 y = 0;
                                 height = adjusted_mode->vdisplay;
                         } else if (scaled_width < scaled_height) { /* letter */
                                 height = scaled_width / mode->hdisplay;
+                                if (height & 1)
+                                    height++;
                                 y = (adjusted_mode->vdisplay - height + 1) / 2;
                                 x = 0;
                                 width = adjusted_mode->hdisplay;
@@ -222,7 +226,7 @@ static void intel_pch_panel_set_backlight(struct drm_device *dev, u32 level)
         I915_WRITE(BLC_PWM_CPU_CTL, val | level);
 }
 
-void intel_panel_set_backlight(struct drm_device *dev, u32 level)
+static void intel_panel_actually_set_backlight(struct drm_device *dev, u32 level)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 tmp;
@@ -250,16 +254,21 @@ void intel_panel_set_backlight(struct drm_device *dev, u32 level)
         I915_WRITE(BLC_PWM_CTL, tmp | level);
 }
 
-void intel_panel_disable_backlight(struct drm_device *dev)
+void intel_panel_set_backlight(struct drm_device *dev, u32 level)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (dev_priv->backlight_enabled) {
-                dev_priv->backlight_level = intel_panel_get_backlight(dev);
-                dev_priv->backlight_enabled = false;
-        }
+        dev_priv->backlight_level = level;
+        if (dev_priv->backlight_enabled)
+                intel_panel_actually_set_backlight(dev, level);
+}
+
+void intel_panel_disable_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
-        intel_panel_set_backlight(dev, 0);
+        dev_priv->backlight_enabled = false;
+        intel_panel_actually_set_backlight(dev, 0);
 }
 
 void intel_panel_enable_backlight(struct drm_device *dev)
@@ -269,11 +278,11 @@ void intel_panel_enable_backlight(struct drm_device *dev)
         if (dev_priv->backlight_level == 0)
                 dev_priv->backlight_level = intel_panel_get_max_backlight(dev);
 
-        intel_panel_set_backlight(dev, dev_priv->backlight_level);
         dev_priv->backlight_enabled = true;
+        intel_panel_actually_set_backlight(dev, dev_priv->backlight_level);
 }
 
-void intel_panel_setup_backlight(struct drm_device *dev)
+static void intel_panel_init_backlight(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -305,3 +314,74 @@ intel_panel_detect(struct drm_device *dev)
 
         return connector_status_unknown;
 }
+
+#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+static int intel_panel_update_status(struct backlight_device *bd)
+{
+        struct drm_device *dev = bl_get_data(bd);
+        intel_panel_set_backlight(dev, bd->props.brightness);
+        return 0;
+}
+
+static int intel_panel_get_brightness(struct backlight_device *bd)
+{
+        struct drm_device *dev = bl_get_data(bd);
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        return dev_priv->backlight_level;
+}
+
+static const struct backlight_ops intel_panel_bl_ops = {
+        .update_status = intel_panel_update_status,
+        .get_brightness = intel_panel_get_brightness,
+};
+
+int intel_panel_setup_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct backlight_properties props;
+        struct drm_connector *connector;
+
+        intel_panel_init_backlight(dev);
+
+        if (dev_priv->int_lvds_connector)
+                connector = dev_priv->int_lvds_connector;
+        else if (dev_priv->int_edp_connector)
+                connector = dev_priv->int_edp_connector;
+        else
+                return -ENODEV;
+
+        props.type = BACKLIGHT_RAW;
+        props.max_brightness = intel_panel_get_max_backlight(dev);
+        dev_priv->backlight =
+                backlight_device_register("intel_backlight",
+                                          &connector->kdev, dev,
+                                          &intel_panel_bl_ops, &props);
+
+        if (IS_ERR(dev_priv->backlight)) {
+                DRM_ERROR("Failed to register backlight: %ld\n",
+                          PTR_ERR(dev_priv->backlight));
+                dev_priv->backlight = NULL;
+                return -ENODEV;
+        }
+        dev_priv->backlight->props.brightness = intel_panel_get_backlight(dev);
+        return 0;
+}
+
+void intel_panel_destroy_backlight(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        if (dev_priv->backlight)
+                backlight_device_unregister(dev_priv->backlight);
+}
+#else
+int intel_panel_setup_backlight(struct drm_device *dev)
+{
+        intel_panel_init_backlight(dev);
+        return 0;
+}
+
+void intel_panel_destroy_backlight(struct drm_device *dev)
+{
+        return;
+}
+#endif
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 95c4b142993..c30626ea9f9 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -236,7 +236,8 @@ init_pipe_control(struct intel_ring_buffer *ring)
                 ret = -ENOMEM;
                 goto err;
         }
-        obj->cache_level = I915_CACHE_LLC;
+
+        i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
         ret = i915_gem_object_pin(obj, 4096, true);
         if (ret)
@@ -289,6 +290,10 @@ static int init_render_ring(struct intel_ring_buffer *ring)
                 if (IS_GEN6(dev) || IS_GEN7(dev))
                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
                 I915_WRITE(MI_MODE, mode);
+                if (IS_GEN7(dev))
+                        I915_WRITE(GFX_MODE_GEN7,
+                                   GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
+                                   GFX_MODE_ENABLE(GFX_REPLAY_MODE));
         }
 
         if (INTEL_INFO(dev)->gen >= 6) {
@@ -776,7 +781,8 @@ static int init_status_page(struct intel_ring_buffer *ring)
                 ret = -ENOMEM;
                 goto err;
         }
-        obj->cache_level = I915_CACHE_LLC;
+
+        i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
         ret = i915_gem_object_pin(obj, 4096, true);
         if (ret != 0) {
@@ -1319,6 +1325,9 @@ int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
                 ring->get_seqno = pc_render_get_seqno;
         }
 
+        if (!I915_NEED_GFX_HWS(dev))
+                ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
+
         ring->dev = dev;
         INIT_LIST_HEAD(&ring->active_list);
         INIT_LIST_HEAD(&ring->request_list);
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 30fe554d893..66e47a0100c 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -92,6 +92,11 @@ struct intel_sdvo {
         */
         uint16_t attached_output;
 
+        /*
+         * Hotplug activation bits for this device
+         */
+        uint8_t hotplug_active[2];
+
         /**
          * This is used to select the color range of RBG outputs in HDMI mode.
          * It is only valid when using TMDS encoding and 8 bit per color mode.
@@ -1059,15 +1064,13 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
 
         /* Set the SDVO control regs. */
         if (INTEL_INFO(dev)->gen >= 4) {
-                sdvox = 0;
+                /* The real mode polarity is set by the SDVO commands, using
+                 * struct intel_sdvo_dtd. */
+                sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH;
                 if (intel_sdvo->is_hdmi)
                         sdvox |= intel_sdvo->color_range;
                 if (INTEL_INFO(dev)->gen < 5)
                         sdvox |= SDVO_BORDER_ENABLE;
-                if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
-                        sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
-                if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
-                        sdvox |= SDVO_HSYNC_ACTIVE_HIGH;
         } else {
                 sdvox = I915_READ(intel_sdvo->sdvo_reg);
                 switch (intel_sdvo->sdvo_reg) {
@@ -1080,8 +1083,12 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
                 }
                 sdvox |= (9 << 19) | SDVO_BORDER_ENABLE;
         }
-        if (intel_crtc->pipe == 1)
-                sdvox |= SDVO_PIPE_B_SELECT;
+
+        if (INTEL_PCH_TYPE(dev) >= PCH_CPT)
+                sdvox |= TRANSCODER_CPT(intel_crtc->pipe);
+        else
+                sdvox |= TRANSCODER(intel_crtc->pipe);
+
         if (intel_sdvo->has_hdmi_audio)
                 sdvox |= SDVO_AUDIO_ENABLE;
 
@@ -1208,74 +1215,20 @@ static bool intel_sdvo_get_capabilities(struct intel_sdvo *intel_sdvo, struct in
         return true;
 }
 
-/* No use! */
-#if 0
-struct drm_connector* intel_sdvo_find(struct drm_device *dev, int sdvoB)
-{
-        struct drm_connector *connector = NULL;
-        struct intel_sdvo *iout = NULL;
-        struct intel_sdvo *sdvo;
-
-        /* find the sdvo connector */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                iout = to_intel_sdvo(connector);
-
-                if (iout->type != INTEL_OUTPUT_SDVO)
-                        continue;
-
-                sdvo = iout->dev_priv;
-
-                if (sdvo->sdvo_reg == SDVOB && sdvoB)
-                        return connector;
-
-                if (sdvo->sdvo_reg == SDVOC && !sdvoB)
-                        return connector;
-
-        }
-
-        return NULL;
-}
-
-int intel_sdvo_supports_hotplug(struct drm_connector *connector)
+static int intel_sdvo_supports_hotplug(struct intel_sdvo *intel_sdvo)
 {
         u8 response[2];
-        u8 status;
-        struct intel_sdvo *intel_sdvo;
-        DRM_DEBUG_KMS("\n");
-
-        if (!connector)
-                return 0;
-
-        intel_sdvo = to_intel_sdvo(connector);
 
         return intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_HOT_PLUG_SUPPORT,
                                     &response, 2) && response[0];
 }
 
-void intel_sdvo_set_hotplug(struct drm_connector *connector, int on)
+static void intel_sdvo_enable_hotplug(struct intel_encoder *encoder)
 {
-        u8 response[2];
-        u8 status;
-        struct intel_sdvo *intel_sdvo = to_intel_sdvo(connector);
-
-        intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_GET_ACTIVE_HOT_PLUG, NULL, 0);
-        intel_sdvo_read_response(intel_sdvo, &response, 2);
+        struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
 
-        if (on) {
-                intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_GET_HOT_PLUG_SUPPORT, NULL, 0);
-                status = intel_sdvo_read_response(intel_sdvo, &response, 2);
-
-                intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG, &response, 2);
-        } else {
-                response[0] = 0;
-                response[1] = 0;
-                intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG, &response, 2);
-        }
-
-        intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_GET_ACTIVE_HOT_PLUG, NULL, 0);
-        intel_sdvo_read_response(intel_sdvo, &response, 2);
+        intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG, &intel_sdvo->hotplug_active, 2);
 }
-#endif
 
 static bool
 intel_sdvo_multifunc_encoder(struct intel_sdvo *intel_sdvo)
@@ -2045,6 +1998,7 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 {
         struct drm_encoder *encoder = &intel_sdvo->base.base;
         struct drm_connector *connector;
+        struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
         struct intel_connector *intel_connector;
         struct intel_sdvo_connector *intel_sdvo_connector;
 
@@ -2062,7 +2016,17 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 
         intel_connector = &intel_sdvo_connector->base;
         connector = &intel_connector->base;
-        connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
+        if (intel_sdvo_supports_hotplug(intel_sdvo) & (1 << device)) {
+                connector->polled = DRM_CONNECTOR_POLL_HPD;
+                intel_sdvo->hotplug_active[0] |= 1 << device;
+                /* Some SDVO devices have one-shot hotplug interrupts.
+                 * Ensure that they get re-enabled when an interrupt happens.
+                 */
+                intel_encoder->hot_plug = intel_sdvo_enable_hotplug;
+                intel_sdvo_enable_hotplug(intel_encoder);
+        }
+        else
+                connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
         encoder->encoder_type = DRM_MODE_ENCODER_TMDS;
         connector->connector_type = DRM_MODE_CONNECTOR_DVID;
 
@@ -2569,6 +2533,14 @@ bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
         if (!intel_sdvo_get_capabilities(intel_sdvo, &intel_sdvo->caps))
                 goto err;
 
+        /* Set up hotplug command - note paranoia about contents of reply.
+         * We assume that the hardware is in a sane state, and only touch
+         * the bits we think we understand.
+         */
+        intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ACTIVE_HOT_PLUG,
+                             &intel_sdvo->hotplug_active, 2);
+        intel_sdvo->hotplug_active[0] &= ~0x3;
+
         if (intel_sdvo_output_setup(intel_sdvo,
                                     intel_sdvo->caps.output_flags) != true) {
                 DRM_DEBUG_KMS("SDVO output failed to setup on SDVO%c\n",
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 113e4e7264c..210d570fd51 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1236,6 +1236,8 @@ intel_tv_detect_type (struct intel_tv *intel_tv,
                       struct drm_connector *connector)
 {
         struct drm_encoder *encoder = &intel_tv->base.base;
+        struct drm_crtc *crtc = encoder->crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_device *dev = encoder->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
@@ -1258,6 +1260,10 @@ intel_tv_detect_type (struct intel_tv *intel_tv,
         /* Poll for TV detection */
         tv_ctl &= ~(TV_ENC_ENABLE | TV_TEST_MODE_MASK);
         tv_ctl |= TV_TEST_MODE_MONITOR_DETECT;
+        if (intel_crtc->pipe == 1)
+                tv_ctl |= TV_ENC_PIPEB_SELECT;
+        else
+                tv_ctl &= ~TV_ENC_PIPEB_SELECT;
 
         tv_dac &= ~(TVDAC_SENSE_MASK | DAC_A_MASK | DAC_B_MASK | DAC_C_MASK);
         tv_dac |= (TVDAC_STATE_CHG_EN |
@@ -1277,26 +1283,26 @@ intel_tv_detect_type (struct intel_tv *intel_tv,
                               to_intel_crtc(intel_tv->base.base.crtc)->pipe);
 
         type = -1;
-        if (wait_for((tv_dac = I915_READ(TV_DAC)) & TVDAC_STATE_CHG, 20) == 0) {
-                DRM_DEBUG_KMS("TV detected: %x, %x\n", tv_ctl, tv_dac);
-                /*
-                 *  A B C
-                 *  0 1 1 Composite
-                 *  1 0 X svideo
-                 *  0 0 0 Component
-                 */
-                if ((tv_dac & TVDAC_SENSE_MASK) == (TVDAC_B_SENSE | TVDAC_C_SENSE)) {
-                        DRM_DEBUG_KMS("Detected Composite TV connection\n");
-                        type = DRM_MODE_CONNECTOR_Composite;
-                } else if ((tv_dac & (TVDAC_A_SENSE|TVDAC_B_SENSE)) == TVDAC_A_SENSE) {
-                        DRM_DEBUG_KMS("Detected S-Video TV connection\n");
-                        type = DRM_MODE_CONNECTOR_SVIDEO;
-                } else if ((tv_dac & TVDAC_SENSE_MASK) == 0) {
-                        DRM_DEBUG_KMS("Detected Component TV connection\n");
-                        type = DRM_MODE_CONNECTOR_Component;
-                } else {
-                        DRM_DEBUG_KMS("Unrecognised TV connection\n");
-                }
+        tv_dac = I915_READ(TV_DAC);
+        DRM_DEBUG_KMS("TV detected: %x, %x\n", tv_ctl, tv_dac);
+        /*
+         *  A B C
+         *  0 1 1 Composite
+         *  1 0 X svideo
+         *  0 0 0 Component
+         */
+        if ((tv_dac & TVDAC_SENSE_MASK) == (TVDAC_B_SENSE | TVDAC_C_SENSE)) {
+                DRM_DEBUG_KMS("Detected Composite TV connection\n");
+                type = DRM_MODE_CONNECTOR_Composite;
+        } else if ((tv_dac & (TVDAC_A_SENSE|TVDAC_B_SENSE)) == TVDAC_A_SENSE) {
+                DRM_DEBUG_KMS("Detected S-Video TV connection\n");
+                type = DRM_MODE_CONNECTOR_SVIDEO;
+        } else if ((tv_dac & TVDAC_SENSE_MASK) == 0) {
+                DRM_DEBUG_KMS("Detected Component TV connection\n");
+                type = DRM_MODE_CONNECTOR_Component;
+        } else {
+                DRM_DEBUG_KMS("Unrecognised TV connection\n");
+                type = -1;
         }
 
         I915_WRITE(TV_DAC, save_tv_dac & ~TVDAC_STATE_CHG_EN);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 729d5fd7c88..b311faba34f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -135,13 +135,14 @@ static void load_vbios_pramin(struct drm_device *dev, uint8_t *data)
         int i;
 
         if (dev_priv->card_type >= NV_50) {
-                uint32_t vbios_vram = (nv_rd32(dev, 0x619f04) & ~0xff) << 8;
-
-                if (!vbios_vram)
-                        vbios_vram = (nv_rd32(dev, 0x1700) << 16) + 0xf0000;
+                u64 addr = (u64)(nv_rd32(dev, 0x619f04) & 0xffffff00) << 8;
+                if (!addr) {
+                        addr  = (u64)nv_rd32(dev, 0x1700) << 16;
+                        addr += 0xf0000;
+                }
 
                 old_bar0_pramin = nv_rd32(dev, 0x1700);
-                nv_wr32(dev, 0x1700, vbios_vram >> 16);
+                nv_wr32(dev, 0x1700, addr >> 16);
         }
 
         /* bail if no rom signature */
@@ -5186,7 +5187,7 @@ static int parse_bit_A_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
         load_table_ptr = ROM16(bios->data[bitentry->offset]);
 
         if (load_table_ptr == 0x0) {
-                NV_ERROR(dev, "Pointer to BIT loadval table invalid\n");
+                NV_DEBUG(dev, "Pointer to BIT loadval table invalid\n");
                 return -EINVAL;
         }
 
@@ -5965,6 +5966,12 @@ apply_dcb_connector_quirks(struct nvbios *bios, int idx)
                 if (cte->type == DCB_CONNECTOR_HDMI_1)
                         cte->type = DCB_CONNECTOR_DVI_I;
         }
+
+        /* Gigabyte GV-NX86T512H */
+        if (nv_match_device(dev, 0x0402, 0x1458, 0x3455)) {
+                if (cte->type == DCB_CONNECTOR_HDMI_1)
+                        cte->type = DCB_CONNECTOR_DVI_I;
+        }
 }
 
 static const u8 hpd_gpio[16] = {
@@ -6377,6 +6384,37 @@ apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
                 }
         }
 
+        /* Some other twisted XFX board (rhbz#694914)
+         *
+         * The DVI/VGA encoder combo that's supposed to represent the
+         * DVI-I connector actually point at two different ones, and
+         * the HDMI connector ends up paired with the VGA instead.
+         *
+         * Connector table is missing anything for VGA at all, pointing it
+         * an invalid conntab entry 2 so we figure it out ourself.
+         */
+        if (nv_match_device(dev, 0x0615, 0x1682, 0x2605)) {
+                if (idx == 0) {
+                        *conn = 0x02002300; /* VGA, connector 2 */
+                        *conf = 0x00000028;
+                } else
+                if (idx == 1) {
+                        *conn = 0x01010312; /* DVI, connector 0 */
+                        *conf = 0x00020030;
+                } else
+                if (idx == 2) {
+                        *conn = 0x04020310; /* VGA, connector 0 */
+                        *conf = 0x00000028;
+                } else
+                if (idx == 3) {
+                        *conn = 0x02021322; /* HDMI, connector 1 */
+                        *conf = 0x00020010;
+                } else {
+                        *conn = 0x0000000e; /* EOL */
+                        *conf = 0x00000000;
+                }
+        }
+
         return true;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 2ad49cbf7c8..890d50e4d68 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -49,16 +49,12 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
                 DRM_ERROR("bo %p still attached to GEM object\n", bo);
 
         nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
-        if (nvbo->vma.node) {
-                nouveau_vm_unmap(&nvbo->vma);
-                nouveau_vm_put(&nvbo->vma);
-        }
         kfree(nvbo);
 }
 
 static void
 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
-                       int *align, int *size, int *page_shift)
+                       int *align, int *size)
 {
         struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
 
@@ -82,67 +78,51 @@ nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
                         }
                 }
         } else {
-                if (likely(dev_priv->chan_vm)) {
-                        if (!(flags & TTM_PL_FLAG_TT) &&  *size > 256 * 1024)
-                                *page_shift = dev_priv->chan_vm->lpg_shift;
-                        else
-                                *page_shift = dev_priv->chan_vm->spg_shift;
-                } else {
-                        *page_shift = 12;
-                }
-
-                *size = roundup(*size, (1 << *page_shift));
-                *align = max((1 << *page_shift), *align);
+                *size = roundup(*size, (1 << nvbo->page_shift));
+                *align = max((1 <<  nvbo->page_shift), *align);
         }
 
         *size = roundup(*size, PAGE_SIZE);
 }
 
 int
-nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
-               int size, int align, uint32_t flags, uint32_t tile_mode,
-               uint32_t tile_flags, struct nouveau_bo **pnvbo)
+nouveau_bo_new(struct drm_device *dev, int size, int align,
+               uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
+               struct nouveau_bo **pnvbo)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_bo *nvbo;
-        int ret = 0, page_shift = 0;
+        int ret;
 
         nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
         if (!nvbo)
                 return -ENOMEM;
         INIT_LIST_HEAD(&nvbo->head);
         INIT_LIST_HEAD(&nvbo->entry);
+        INIT_LIST_HEAD(&nvbo->vma_list);
         nvbo->tile_mode = tile_mode;
         nvbo->tile_flags = tile_flags;
         nvbo->bo.bdev = &dev_priv->ttm.bdev;
 
-        nouveau_bo_fixup_align(nvbo, flags, &align, &size, &page_shift);
-        align >>= PAGE_SHIFT;
-
-        if (dev_priv->chan_vm) {
-                ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift,
-                                     NV_MEM_ACCESS_RW, &nvbo->vma);
-                if (ret) {
-                        kfree(nvbo);
-                        return ret;
-                }
+        nvbo->page_shift = 12;
+        if (dev_priv->bar1_vm) {
+                if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
+                        nvbo->page_shift = dev_priv->bar1_vm->lpg_shift;
         }
 
+        nouveau_bo_fixup_align(nvbo, flags, &align, &size);
         nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
         nouveau_bo_placement_set(nvbo, flags, 0);
 
-        nvbo->channel = chan;
         ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
-                          ttm_bo_type_device, &nvbo->placement, align, 0,
-                          false, NULL, size, nouveau_bo_del_ttm);
+                          ttm_bo_type_device, &nvbo->placement,
+                          align >> PAGE_SHIFT, 0, false, NULL, size,
+                          nouveau_bo_del_ttm);
         if (ret) {
                 /* ttm will call nouveau_bo_del_ttm if it fails.. */
                 return ret;
         }
-        nvbo->channel = NULL;
 
-        if (nvbo->vma.node)
-                nvbo->bo.offset = nvbo->vma.offset;
         *pnvbo = nvbo;
         return 0;
 }
@@ -312,8 +292,6 @@ nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
         if (ret)
                 return ret;
 
-        if (nvbo->vma.node)
-                nvbo->bo.offset = nvbo->vma.offset;
         return 0;
 }
 
@@ -440,7 +418,6 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                                      TTM_MEMTYPE_FLAG_CMA;
                         man->available_caching = TTM_PL_MASK_CACHING;
                         man->default_caching = TTM_PL_FLAG_CACHED;
-                        man->gpu_offset = dev_priv->gart_info.aper_base;
                         break;
                 default:
                         NV_ERROR(dev, "Unknown GART type: %d\n",
@@ -501,19 +478,12 @@ static int
 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
-        struct nouveau_mem *old_node = old_mem->mm_node;
-        struct nouveau_mem *new_node = new_mem->mm_node;
-        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        struct nouveau_mem *node = old_mem->mm_node;
+        u64 src_offset = node->vma[0].offset;
+        u64 dst_offset = node->vma[1].offset;
         u32 page_count = new_mem->num_pages;
-        u64 src_offset, dst_offset;
         int ret;
 
-        src_offset = old_node->tmp_vma.offset;
-        if (new_node->tmp_vma.node)
-                dst_offset = new_node->tmp_vma.offset;
-        else
-                dst_offset = nvbo->vma.offset;
-
         page_count = new_mem->num_pages;
         while (page_count) {
                 int line_count = (page_count > 2047) ? 2047 : page_count;
@@ -547,19 +517,13 @@ static int
 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
                   struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
-        struct nouveau_mem *old_node = old_mem->mm_node;
-        struct nouveau_mem *new_node = new_mem->mm_node;
+        struct nouveau_mem *node = old_mem->mm_node;
         struct nouveau_bo *nvbo = nouveau_bo(bo);
         u64 length = (new_mem->num_pages << PAGE_SHIFT);
-        u64 src_offset, dst_offset;
+        u64 src_offset = node->vma[0].offset;
+        u64 dst_offset = node->vma[1].offset;
         int ret;
 
-        src_offset = old_node->tmp_vma.offset;
-        if (new_node->tmp_vma.node)
-                dst_offset = new_node->tmp_vma.offset;
-        else
-                dst_offset = nvbo->vma.offset;
-
         while (length) {
                 u32 amount, stride, height;
 
@@ -695,6 +659,27 @@ nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 }
 
 static int
+nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo,
+                   struct ttm_mem_reg *mem, struct nouveau_vma *vma)
+{
+        struct nouveau_mem *node = mem->mm_node;
+        int ret;
+
+        ret = nouveau_vm_get(chan->vm, mem->num_pages << PAGE_SHIFT,
+                             node->page_shift, NV_MEM_ACCESS_RO, vma);
+        if (ret)
+                return ret;
+
+        if (mem->mem_type == TTM_PL_VRAM)
+                nouveau_vm_map(vma, node);
+        else
+                nouveau_vm_map_sg(vma, 0, mem->num_pages << PAGE_SHIFT,
+                                  node, node->pages);
+
+        return 0;
+}
+
+static int
 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                      bool no_wait_reserve, bool no_wait_gpu,
                      struct ttm_mem_reg *new_mem)
@@ -711,31 +696,20 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                 mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
         }
 
-        /* create temporary vma for old memory, this will get cleaned
-         * up after ttm destroys the ttm_mem_reg
+        /* create temporary vmas for the transfer and attach them to the
+         * old nouveau_mem node, these will get cleaned up after ttm has
+         * destroyed the ttm_mem_reg
          */
         if (dev_priv->card_type >= NV_50) {
                 struct nouveau_mem *node = old_mem->mm_node;
-                if (!node->tmp_vma.node) {
-                        u32 page_shift = nvbo->vma.node->type;
-                        if (old_mem->mem_type == TTM_PL_TT)
-                                page_shift = nvbo->vma.vm->spg_shift;
-
-                        ret = nouveau_vm_get(chan->vm,
-                                             old_mem->num_pages << PAGE_SHIFT,
-                                             page_shift, NV_MEM_ACCESS_RO,
-                                             &node->tmp_vma);
-                        if (ret)
-                                goto out;
-                }
 
-                if (old_mem->mem_type == TTM_PL_VRAM)
-                        nouveau_vm_map(&node->tmp_vma, node);
-                else {
-                        nouveau_vm_map_sg(&node->tmp_vma, 0,
-                                          old_mem->num_pages << PAGE_SHIFT,
-                                          node, node->pages);
-                }
+                ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]);
+                if (ret)
+                        goto out;
+
+                ret = nouveau_vma_getmap(chan, nvbo, new_mem, &node->vma[1]);
+                if (ret)
+                        goto out;
         }
 
         if (dev_priv->card_type < NV_50)
@@ -762,7 +736,6 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
                       bool no_wait_reserve, bool no_wait_gpu,
                       struct ttm_mem_reg *new_mem)
 {
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
         u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
         struct ttm_placement placement;
         struct ttm_mem_reg tmp_mem;
@@ -782,23 +755,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
         if (ret)
                 goto out;
 
-        if (dev_priv->card_type >= NV_50) {
-                struct nouveau_bo *nvbo = nouveau_bo(bo);
-                struct nouveau_mem *node = tmp_mem.mm_node;
-                struct nouveau_vma *vma = &nvbo->vma;
-                if (vma->node->type != vma->vm->spg_shift)
-                        vma = &node->tmp_vma;
-                nouveau_vm_map_sg(vma, 0, tmp_mem.num_pages << PAGE_SHIFT,
-                                  node, node->pages);
-        }
-
         ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
-
-        if (dev_priv->card_type >= NV_50) {
-                struct nouveau_bo *nvbo = nouveau_bo(bo);
-                nouveau_vm_unmap(&nvbo->vma);
-        }
-
         if (ret)
                 goto out;
 
@@ -844,30 +801,22 @@ out:
 static void
 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
 {
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
         struct nouveau_mem *node = new_mem->mm_node;
         struct nouveau_bo *nvbo = nouveau_bo(bo);
-        struct nouveau_vma *vma = &nvbo->vma;
-        struct nouveau_vm *vm = vma->vm;
-
-        if (dev_priv->card_type < NV_50)
-                return;
-
-        switch (new_mem->mem_type) {
-        case TTM_PL_VRAM:
-                nouveau_vm_map(vma, node);
-                break;
-        case TTM_PL_TT:
-                if (vma->node->type != vm->spg_shift) {
+        struct nouveau_vma *vma;
+
+        list_for_each_entry(vma, &nvbo->vma_list, head) {
+                if (new_mem->mem_type == TTM_PL_VRAM) {
+                        nouveau_vm_map(vma, new_mem->mm_node);
+                } else
+                if (new_mem->mem_type == TTM_PL_TT &&
+                    nvbo->page_shift == vma->vm->spg_shift) {
+                        nouveau_vm_map_sg(vma, 0, new_mem->
+                                          num_pages << PAGE_SHIFT,
+                                          node, node->pages);
+                } else {
                         nouveau_vm_unmap(vma);
-                        vma = &node->tmp_vma;
                 }
-                nouveau_vm_map_sg(vma, 0, new_mem->num_pages << PAGE_SHIFT,
-                                  node, node->pages);
-                break;
-        default:
-                nouveau_vm_unmap(&nvbo->vma);
-                break;
         }
 }
 
@@ -1113,3 +1062,54 @@ struct ttm_bo_driver nouveau_bo_driver = {
         .io_mem_free = &nouveau_ttm_io_mem_free,
 };
 
+struct nouveau_vma *
+nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm)
+{
+        struct nouveau_vma *vma;
+        list_for_each_entry(vma, &nvbo->vma_list, head) {
+                if (vma->vm == vm)
+                        return vma;
+        }
+
+        return NULL;
+}
+
+int
+nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm,
+                   struct nouveau_vma *vma)
+{
+        const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
+        struct nouveau_mem *node = nvbo->bo.mem.mm_node;
+        int ret;
+
+        ret = nouveau_vm_get(vm, size, nvbo->page_shift,
+                             NV_MEM_ACCESS_RW, vma);
+        if (ret)
+                return ret;
+
+        if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
+                nouveau_vm_map(vma, nvbo->bo.mem.mm_node);
+        else
+        if (nvbo->bo.mem.mem_type == TTM_PL_TT)
+                nouveau_vm_map_sg(vma, 0, size, node, node->pages);
+
+        list_add_tail(&vma->head, &nvbo->vma_list);
+        vma->refcount = 1;
+        return 0;
+}
+
+void
+nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
+{
+        if (vma->node) {
+                if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) {
+                        spin_lock(&nvbo->bo.bdev->fence_lock);
+                        ttm_bo_wait(&nvbo->bo, false, false, false);
+                        spin_unlock(&nvbo->bo.bdev->fence_lock);
+                        nouveau_vm_unmap(vma);
+                }
+
+                nouveau_vm_put(vma);
+                list_del(&vma->head);
+        }
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_channel.c b/drivers/gpu/drm/nouveau/nouveau_channel.c
index a7583a8ddb0..0e3241c39b8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_channel.c
+++ b/drivers/gpu/drm/nouveau/nouveau_channel.c
@@ -27,40 +27,63 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
+#include "nouveau_ramht.h"
 
 static int
-nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
+nouveau_channel_pushbuf_init(struct nouveau_channel *chan)
 {
+        u32 mem = nouveau_vram_pushbuf ? TTM_PL_FLAG_VRAM : TTM_PL_FLAG_TT;
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_bo *pb = chan->pushbuf_bo;
-        struct nouveau_gpuobj *pushbuf = NULL;
-        int ret = 0;
+        int ret;
+
+        /* allocate buffer object */
+        ret = nouveau_bo_new(dev, 65536, 0, mem, 0, 0, &chan->pushbuf_bo);
+        if (ret)
+                goto out;
+
+        ret = nouveau_bo_pin(chan->pushbuf_bo, mem);
+        if (ret)
+                goto out;
+
+        ret = nouveau_bo_map(chan->pushbuf_bo);
+        if (ret)
+                goto out;
 
+        /* create DMA object covering the entire memtype where the push
+         * buffer resides, userspace can submit its own push buffers from
+         * anywhere within the same memtype.
+         */
+        chan->pushbuf_base = chan->pushbuf_bo->bo.offset;
         if (dev_priv->card_type >= NV_50) {
+                ret = nouveau_bo_vma_add(chan->pushbuf_bo, chan->vm,
+                                         &chan->pushbuf_vma);
+                if (ret)
+                        goto out;
+
                 if (dev_priv->card_type < NV_C0) {
                         ret = nouveau_gpuobj_dma_new(chan,
                                                      NV_CLASS_DMA_IN_MEMORY, 0,
                                                      (1ULL << 40),
                                                      NV_MEM_ACCESS_RO,
                                                      NV_MEM_TARGET_VM,
-                                                     &pushbuf);
+                                                     &chan->pushbuf);
                 }
-                chan->pushbuf_base = pb->bo.offset;
+                chan->pushbuf_base = chan->pushbuf_vma.offset;
         } else
-        if (pb->bo.mem.mem_type == TTM_PL_TT) {
+        if (chan->pushbuf_bo->bo.mem.mem_type == TTM_PL_TT) {
                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
                                              dev_priv->gart_info.aper_size,
                                              NV_MEM_ACCESS_RO,
-                                             NV_MEM_TARGET_GART, &pushbuf);
-                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
+                                             NV_MEM_TARGET_GART,
+                                             &chan->pushbuf);
         } else
         if (dev_priv->card_type != NV_04) {
                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
                                              dev_priv->fb_available_size,
                                              NV_MEM_ACCESS_RO,
-                                             NV_MEM_TARGET_VRAM, &pushbuf);
-                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
+                                             NV_MEM_TARGET_VRAM,
+                                             &chan->pushbuf);
         } else {
                 /* NV04 cmdbuf hack, from original ddx.. not sure of it's
                  * exact reason for existing :)  PCI access to cmdbuf in
@@ -70,47 +93,22 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
                                              pci_resource_start(dev->pdev, 1),
                                              dev_priv->fb_available_size,
                                              NV_MEM_ACCESS_RO,
-                                             NV_MEM_TARGET_PCI, &pushbuf);
-                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
+                                             NV_MEM_TARGET_PCI,
+                                             &chan->pushbuf);
         }
 
-        nouveau_gpuobj_ref(pushbuf, &chan->pushbuf);
-        nouveau_gpuobj_ref(NULL, &pushbuf);
-        return ret;
-}
-
-static struct nouveau_bo *
-nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
-{
-        struct nouveau_bo *pushbuf = NULL;
-        int location, ret;
-
-        if (nouveau_vram_pushbuf)
-                location = TTM_PL_FLAG_VRAM;
-        else
-                location = TTM_PL_FLAG_TT;
-
-        ret = nouveau_bo_new(dev, NULL, 65536, 0, location, 0, 0x0000, &pushbuf);
-        if (ret) {
-                NV_ERROR(dev, "error allocating DMA push buffer: %d\n", ret);
-                return NULL;
-        }
-
-        ret = nouveau_bo_pin(pushbuf, location);
-        if (ret) {
-                NV_ERROR(dev, "error pinning DMA push buffer: %d\n", ret);
-                nouveau_bo_ref(NULL, &pushbuf);
-                return NULL;
-        }
-
-        ret = nouveau_bo_map(pushbuf);
+out:
         if (ret) {
-                nouveau_bo_unpin(pushbuf);
-                nouveau_bo_ref(NULL, &pushbuf);
-                return NULL;
+                NV_ERROR(dev, "error initialising pushbuf: %d\n", ret);
+                nouveau_bo_vma_del(chan->pushbuf_bo, &chan->pushbuf_vma);
+                nouveau_gpuobj_ref(NULL, &chan->pushbuf);
+                if (chan->pushbuf_bo) {
+                        nouveau_bo_unmap(chan->pushbuf_bo);
+                        nouveau_bo_ref(NULL, &chan->pushbuf_bo);
+                }
         }
 
-        return pushbuf;
+        return 0;
 }
 
 /* allocates and initializes a fifo for user space consumption */
@@ -121,6 +119,7 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
         struct nouveau_channel *chan;
         unsigned long flags;
         int ret;
@@ -159,20 +158,16 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
         INIT_LIST_HEAD(&chan->nvsw.vbl_wait);
         INIT_LIST_HEAD(&chan->nvsw.flip);
         INIT_LIST_HEAD(&chan->fence.pending);
+        spin_lock_init(&chan->fence.lock);
 
-        /* Allocate DMA push buffer */
-        chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev);
-        if (!chan->pushbuf_bo) {
-                ret = -ENOMEM;
-                NV_ERROR(dev, "pushbuf %d\n", ret);
+        /* setup channel's memory and vm */
+        ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
+        if (ret) {
+                NV_ERROR(dev, "gpuobj %d\n", ret);
                 nouveau_channel_put(&chan);
                 return ret;
         }
 
-        nouveau_dma_pre_init(chan);
-        chan->user_put = 0x40;
-        chan->user_get = 0x44;
-
         /* Allocate space for per-channel fixed notifier memory */
         ret = nouveau_notifier_init_channel(chan);
         if (ret) {
@@ -181,21 +176,17 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
                 return ret;
         }
 
-        /* Setup channel's default objects */
-        ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
+        /* Allocate DMA push buffer */
+        ret = nouveau_channel_pushbuf_init(chan);
         if (ret) {
-                NV_ERROR(dev, "gpuobj %d\n", ret);
+                NV_ERROR(dev, "pushbuf %d\n", ret);
                 nouveau_channel_put(&chan);
                 return ret;
         }
 
-        /* Create a dma object for the push buffer */
-        ret = nouveau_channel_pushbuf_ctxdma_init(chan);
-        if (ret) {
-                NV_ERROR(dev, "pbctxdma %d\n", ret);
-                nouveau_channel_put(&chan);
-                return ret;
-        }
+        nouveau_dma_pre_init(chan);
+        chan->user_put = 0x40;
+        chan->user_get = 0x44;
 
         /* disable the fifo caches */
         pfifo->reassign(dev, false);
@@ -220,6 +211,11 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
         nouveau_debugfs_channel_init(chan);
 
         NV_DEBUG(dev, "channel %d initialised\n", chan->id);
+        if (fpriv) {
+                spin_lock(&fpriv->lock);
+                list_add(&chan->list, &fpriv->channels);
+                spin_unlock(&fpriv->lock);
+        }
         *chan_ret = chan;
         return 0;
 }
@@ -236,29 +232,23 @@ nouveau_channel_get_unlocked(struct nouveau_channel *ref)
 }
 
 struct nouveau_channel *
-nouveau_channel_get(struct drm_device *dev, struct drm_file *file_priv, int id)
+nouveau_channel_get(struct drm_file *file_priv, int id)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
         struct nouveau_channel *chan;
-        unsigned long flags;
-
-        if (unlikely(id < 0 || id >= NOUVEAU_MAX_CHANNEL_NR))
-                return ERR_PTR(-EINVAL);
-
-        spin_lock_irqsave(&dev_priv->channels.lock, flags);
-        chan = nouveau_channel_get_unlocked(dev_priv->channels.ptr[id]);
-        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-
-        if (unlikely(!chan))
-                return ERR_PTR(-EINVAL);
 
-        if (unlikely(file_priv && chan->file_priv != file_priv)) {
-                nouveau_channel_put_unlocked(&chan);
-                return ERR_PTR(-EINVAL);
+        spin_lock(&fpriv->lock);
+        list_for_each_entry(chan, &fpriv->channels, list) {
+                if (chan->id == id) {
+                        chan = nouveau_channel_get_unlocked(chan);
+                        spin_unlock(&fpriv->lock);
+                        mutex_lock(&chan->mutex);
+                        return chan;
+                }
         }
+        spin_unlock(&fpriv->lock);
 
-        mutex_lock(&chan->mutex);
-        return chan;
+        return ERR_PTR(-EINVAL);
 }
 
 void
@@ -312,12 +302,14 @@ nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
         /* destroy any resources the channel owned */
         nouveau_gpuobj_ref(NULL, &chan->pushbuf);
         if (chan->pushbuf_bo) {
+                nouveau_bo_vma_del(chan->pushbuf_bo, &chan->pushbuf_vma);
                 nouveau_bo_unmap(chan->pushbuf_bo);
                 nouveau_bo_unpin(chan->pushbuf_bo);
                 nouveau_bo_ref(NULL, &chan->pushbuf_bo);
         }
-        nouveau_gpuobj_channel_takedown(chan);
+        nouveau_ramht_ref(NULL, &chan->ramht, chan);
         nouveau_notifier_takedown_channel(chan);
+        nouveau_gpuobj_channel_takedown(chan);
 
         nouveau_channel_ref(NULL, pchan);
 }
@@ -383,10 +375,11 @@ nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv)
 
         NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
         for (i = 0; i < engine->fifo.channels; i++) {
-                chan = nouveau_channel_get(dev, file_priv, i);
+                chan = nouveau_channel_get(file_priv, i);
                 if (IS_ERR(chan))
                         continue;
 
+                list_del(&chan->list);
                 atomic_dec(&chan->users);
                 nouveau_channel_put(&chan);
         }
@@ -459,10 +452,11 @@ nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
         struct drm_nouveau_channel_free *req = data;
         struct nouveau_channel *chan;
 
-        chan = nouveau_channel_get(dev, file_priv, req->channel);
+        chan = nouveau_channel_get(file_priv, req->channel);
         if (IS_ERR(chan))
                 return PTR_ERR(chan);
 
+        list_del(&chan->list);
         atomic_dec(&chan->users);
         nouveau_channel_put(&chan);
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 1595d0b6e81..939d4df0777 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -40,7 +40,7 @@
 static void nouveau_connector_hotplug(void *, int);
 
 static struct nouveau_encoder *
-find_encoder_by_type(struct drm_connector *connector, int type)
+find_encoder(struct drm_connector *connector, int type)
 {
         struct drm_device *dev = connector->dev;
         struct nouveau_encoder *nv_encoder;
@@ -170,8 +170,8 @@ nouveau_connector_of_detect(struct drm_connector *connector)
         struct device_node *cn, *dn = pci_device_to_OF_node(dev->pdev);
 
         if (!dn ||
-            !((nv_encoder = find_encoder_by_type(connector, OUTPUT_TMDS)) ||
-              (nv_encoder = find_encoder_by_type(connector, OUTPUT_ANALOG))))
+            !((nv_encoder = find_encoder(connector, OUTPUT_TMDS)) ||
+              (nv_encoder = find_encoder(connector, OUTPUT_ANALOG))))
                 return NULL;
 
         for_each_child_of_node(dn, cn) {
@@ -233,6 +233,7 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
         struct drm_device *dev = connector->dev;
         struct nouveau_connector *nv_connector = nouveau_connector(connector);
         struct nouveau_encoder *nv_encoder = NULL;
+        struct nouveau_encoder *nv_partner;
         struct nouveau_i2c_chan *i2c;
         int type;
 
@@ -266,19 +267,22 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
                  * same i2c channel so the value returned from ddc_detect
                  * isn't necessarily correct.
                  */
-                if (nv_connector->dcb->type == DCB_CONNECTOR_DVI_I) {
+                nv_partner = NULL;
+                if (nv_encoder->dcb->type == OUTPUT_TMDS)
+                        nv_partner = find_encoder(connector, OUTPUT_ANALOG);
+                if (nv_encoder->dcb->type == OUTPUT_ANALOG)
+                        nv_partner = find_encoder(connector, OUTPUT_TMDS);
+
+                if (nv_partner && ((nv_encoder->dcb->type == OUTPUT_ANALOG &&
+                                    nv_partner->dcb->type == OUTPUT_TMDS) ||
+                                   (nv_encoder->dcb->type == OUTPUT_TMDS &&
+                                    nv_partner->dcb->type == OUTPUT_ANALOG))) {
                         if (nv_connector->edid->input & DRM_EDID_INPUT_DIGITAL)
                                 type = OUTPUT_TMDS;
                         else
                                 type = OUTPUT_ANALOG;
 
-                        nv_encoder = find_encoder_by_type(connector, type);
-                        if (!nv_encoder) {
-                                NV_ERROR(dev, "Detected %d encoder on %s, "
-                                              "but no object!\n", type,
-                                         drm_get_connector_name(connector));
-                                return connector_status_disconnected;
-                        }
+                        nv_encoder = find_encoder(connector, type);
                 }
 
                 nouveau_connector_set_encoder(connector, nv_encoder);
@@ -292,9 +296,9 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
         }
 
 detect_analog:
-        nv_encoder = find_encoder_by_type(connector, OUTPUT_ANALOG);
+        nv_encoder = find_encoder(connector, OUTPUT_ANALOG);
         if (!nv_encoder && !nouveau_tv_disable)
-                nv_encoder = find_encoder_by_type(connector, OUTPUT_TV);
+                nv_encoder = find_encoder(connector, OUTPUT_TV);
         if (nv_encoder && force) {
                 struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
                 struct drm_encoder_helper_funcs *helper =
@@ -327,7 +331,7 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
                 nv_connector->edid = NULL;
         }
 
-        nv_encoder = find_encoder_by_type(connector, OUTPUT_LVDS);
+        nv_encoder = find_encoder(connector, OUTPUT_LVDS);
         if (!nv_encoder)
                 return connector_status_disconnected;
 
@@ -405,7 +409,7 @@ nouveau_connector_force(struct drm_connector *connector)
         } else
                 type = OUTPUT_ANY;
 
-        nv_encoder = find_encoder_by_type(connector, type);
+        nv_encoder = find_encoder(connector, type);
         if (!nv_encoder) {
                 NV_ERROR(connector->dev, "can't find encoder to force %s on!\n",
                          drm_get_connector_name(connector));
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index 568caedd721..00bc6eaad55 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -167,8 +167,13 @@ nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
               int delta, int length)
 {
         struct nouveau_bo *pb = chan->pushbuf_bo;
-        uint64_t offset = bo->bo.offset + delta;
+        struct nouveau_vma *vma;
         int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;
+        u64 offset;
+
+        vma = nouveau_bo_vma_find(bo, chan->vm);
+        BUG_ON(!vma);
+        offset = vma->offset + delta;
 
         BUG_ON(chan->dma.ib_free < 1);
         nouveau_bo_wr32(pb, ip++, lower_32_bits(offset));
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.c b/drivers/gpu/drm/nouveau/nouveau_drv.c
index 02c6f37d8bd..b30ddd8d2e2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.c
@@ -73,7 +73,7 @@ int nouveau_ignorelid = 0;
 module_param_named(ignorelid, nouveau_ignorelid, int, 0400);
 
 MODULE_PARM_DESC(noaccel, "Disable all acceleration");
-int nouveau_noaccel = 0;
+int nouveau_noaccel = -1;
 module_param_named(noaccel, nouveau_noaccel, int, 0400);
 
 MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
@@ -119,6 +119,10 @@ MODULE_PARM_DESC(msi, "Enable MSI (default: off)\n");
 int nouveau_msi;
 module_param_named(msi, nouveau_msi, int, 0400);
 
+MODULE_PARM_DESC(ctxfw, "Use external HUB/GPC ucode (fermi)\n");
+int nouveau_ctxfw;
+module_param_named(ctxfw, nouveau_ctxfw, int, 0400);
+
 int nouveau_fbpercrtc;
 #if 0
 module_param_named(fbpercrtc, nouveau_fbpercrtc, int, 0400);
@@ -210,10 +214,13 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
         pfifo->unload_context(dev);
 
         for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
-                if (dev_priv->eng[e]) {
-                        ret = dev_priv->eng[e]->fini(dev, e);
-                        if (ret)
-                                goto out_abort;
+                if (!dev_priv->eng[e])
+                        continue;
+
+                ret = dev_priv->eng[e]->fini(dev, e, true);
+                if (ret) {
+                        NV_ERROR(dev, "... engine %d failed: %d\n", i, ret);
+                        goto out_abort;
                 }
         }
 
@@ -354,7 +361,7 @@ nouveau_pci_resume(struct pci_dev *pdev)
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-                u32 offset = nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT;
+                u32 offset = nv_crtc->cursor.nvbo->bo.offset;
 
                 nv_crtc->cursor.set_offset(nv_crtc, offset);
                 nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
@@ -389,7 +396,9 @@ static struct drm_driver driver = {
         .firstopen = nouveau_firstopen,
         .lastclose = nouveau_lastclose,
         .unload = nouveau_unload,
+        .open = nouveau_open,
         .preclose = nouveau_preclose,
+        .postclose = nouveau_postclose,
 #if defined(CONFIG_DRM_NOUVEAU_DEBUG)
         .debugfs_init = nouveau_debugfs_init,
         .debugfs_cleanup = nouveau_debugfs_takedown,
@@ -420,6 +429,8 @@ static struct drm_driver driver = {
 
         .gem_init_object = nouveau_gem_object_new,
         .gem_free_object = nouveau_gem_object_del,
+        .gem_open_object = nouveau_gem_object_open,
+        .gem_close_object = nouveau_gem_object_close,
 
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index 9c56331941e..d7d51deb34b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -46,9 +46,17 @@
 #include "ttm/ttm_module.h"
 
 struct nouveau_fpriv {
-        struct ttm_object_file *tfile;
+        spinlock_t lock;
+        struct list_head channels;
+        struct nouveau_vm *vm;
 };
 
+static inline struct nouveau_fpriv *
+nouveau_fpriv(struct drm_file *file_priv)
+{
+        return file_priv ? file_priv->driver_priv : NULL;
+}
+
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 
 #include "nouveau_drm.h"
@@ -69,7 +77,7 @@ struct nouveau_mem {
         struct drm_device *dev;
 
         struct nouveau_vma bar_vma;
-        struct nouveau_vma tmp_vma;
+        struct nouveau_vma vma[2];
         u8  page_shift;
 
         struct drm_mm_node *tag;
@@ -107,7 +115,8 @@ struct nouveau_bo {
 
         struct nouveau_channel *channel;
 
-        struct nouveau_vma vma;
+        struct list_head vma_list;
+        unsigned page_shift;
 
         uint32_t tile_mode;
         uint32_t tile_flags;
@@ -176,9 +185,10 @@ struct nouveau_gpuobj {
         uint32_t flags;
 
         u32 size;
-        u32 pinst;
-        u32 cinst;
-        u64 vinst;
+        u32 pinst;      /* PRAMIN BAR offset */
+        u32 cinst;      /* Channel offset */
+        u64 vinst;      /* VRAM address */
+        u64 linst;      /* VM address */
 
         uint32_t engine;
         uint32_t class;
@@ -201,6 +211,7 @@ enum nouveau_channel_mutex_class {
 
 struct nouveau_channel {
         struct drm_device *dev;
+        struct list_head list;
         int id;
 
         /* references to the channel data structure */
@@ -228,15 +239,18 @@ struct nouveau_channel {
                 uint32_t sequence;
                 uint32_t sequence_ack;
                 atomic_t last_sequence_irq;
+                struct nouveau_vma vma;
         } fence;
 
         /* DMA push buffer */
         struct nouveau_gpuobj *pushbuf;
         struct nouveau_bo     *pushbuf_bo;
+        struct nouveau_vma     pushbuf_vma;
         uint32_t               pushbuf_base;
 
         /* Notifier memory */
         struct nouveau_bo *notifier_bo;
+        struct nouveau_vma notifier_vma;
         struct drm_mm notifier_heap;
 
         /* PFIFO context */
@@ -278,6 +292,7 @@ struct nouveau_channel {
 
         uint32_t sw_subchannel[8];
 
+        struct nouveau_vma dispc_vma[2];
         struct {
                 struct nouveau_gpuobj *vblsem;
                 uint32_t vblsem_head;
@@ -297,7 +312,7 @@ struct nouveau_channel {
 struct nouveau_exec_engine {
         void (*destroy)(struct drm_device *, int engine);
         int  (*init)(struct drm_device *, int engine);
-        int  (*fini)(struct drm_device *, int engine);
+        int  (*fini)(struct drm_device *, int engine, bool suspend);
         int  (*context_new)(struct nouveau_channel *, int engine);
         void (*context_del)(struct nouveau_channel *, int engine);
         int  (*object_new)(struct nouveau_channel *, int engine,
@@ -314,7 +329,8 @@ struct nouveau_instmem_engine {
         int     (*suspend)(struct drm_device *dev);
         void    (*resume)(struct drm_device *dev);
 
-        int     (*get)(struct nouveau_gpuobj *, u32 size, u32 align);
+        int     (*get)(struct nouveau_gpuobj *, struct nouveau_channel *,
+                       u32 size, u32 align);
         void    (*put)(struct nouveau_gpuobj *);
         int     (*map)(struct nouveau_gpuobj *);
         void    (*unmap)(struct nouveau_gpuobj *);
@@ -445,9 +461,9 @@ struct nouveau_pm_level {
 struct nouveau_pm_temp_sensor_constants {
         u16 offset_constant;
         s16 offset_mult;
-        u16 offset_div;
-        u16 slope_mult;
-        u16 slope_div;
+        s16 offset_div;
+        s16 slope_mult;
+        s16 slope_div;
 };
 
 struct nouveau_pm_threshold_temp {
@@ -488,7 +504,10 @@ struct nouveau_pm_engine {
 };
 
 struct nouveau_vram_engine {
+        struct nouveau_mm *mm;
+
         int  (*init)(struct drm_device *);
+        void (*takedown)(struct drm_device *dev);
         int  (*get)(struct drm_device *, u64, u32 align, u32 size_nc,
                     u32 type, struct nouveau_mem **);
         void (*put)(struct drm_device *, struct nouveau_mem **);
@@ -608,6 +627,7 @@ enum nouveau_card_type {
 
 struct drm_nouveau_private {
         struct drm_device *dev;
+        bool noaccel;
 
         /* the card type, takes NV_* as values */
         enum nouveau_card_type card_type;
@@ -700,7 +720,6 @@ struct drm_nouveau_private {
         /* VRAM/fb configuration */
         uint64_t vram_size;
         uint64_t vram_sys_base;
-        u32 vram_rblock_size;
 
         uint64_t fb_phys;
         uint64_t fb_available_size;
@@ -784,12 +803,15 @@ extern int nouveau_override_conntype;
 extern char *nouveau_perflvl;
 extern int nouveau_perflvl_wr;
 extern int nouveau_msi;
+extern int nouveau_ctxfw;
 
 extern int nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state);
 extern int nouveau_pci_resume(struct pci_dev *pdev);
 
 /* nouveau_state.c */
+extern int  nouveau_open(struct drm_device *, struct drm_file *);
 extern void nouveau_preclose(struct drm_device *dev, struct drm_file *);
+extern void nouveau_postclose(struct drm_device *, struct drm_file *);
 extern int  nouveau_load(struct drm_device *, unsigned long flags);
 extern int  nouveau_firstopen(struct drm_device *);
 extern void nouveau_lastclose(struct drm_device *);
@@ -847,7 +869,7 @@ extern int  nouveau_channel_alloc(struct drm_device *dev,
 extern struct nouveau_channel *
 nouveau_channel_get_unlocked(struct nouveau_channel *);
 extern struct nouveau_channel *
-nouveau_channel_get(struct drm_device *, struct drm_file *, int id);
+nouveau_channel_get(struct drm_file *, int id);
 extern void nouveau_channel_put_unlocked(struct nouveau_channel **);
 extern void nouveau_channel_put(struct nouveau_channel **);
 extern void nouveau_channel_ref(struct nouveau_channel *chan,
@@ -1120,7 +1142,6 @@ extern int  nvc0_fifo_unload_context(struct drm_device *);
 
 /* nv04_graph.c */
 extern int  nv04_graph_create(struct drm_device *);
-extern void nv04_graph_fifo_access(struct drm_device *, bool);
 extern int  nv04_graph_object_new(struct nouveau_channel *, int, u32, u16);
 extern int  nv04_graph_mthd_page_flip(struct nouveau_channel *chan,
                                       u32 class, u32 mthd, u32 data);
@@ -1169,7 +1190,8 @@ extern int  nv04_instmem_init(struct drm_device *);
 extern void nv04_instmem_takedown(struct drm_device *);
 extern int  nv04_instmem_suspend(struct drm_device *);
 extern void nv04_instmem_resume(struct drm_device *);
-extern int  nv04_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
+extern int  nv04_instmem_get(struct nouveau_gpuobj *, struct nouveau_channel *,
+                             u32 size, u32 align);
 extern void nv04_instmem_put(struct nouveau_gpuobj *);
 extern int  nv04_instmem_map(struct nouveau_gpuobj *);
 extern void nv04_instmem_unmap(struct nouveau_gpuobj *);
@@ -1180,7 +1202,8 @@ extern int  nv50_instmem_init(struct drm_device *);
 extern void nv50_instmem_takedown(struct drm_device *);
 extern int  nv50_instmem_suspend(struct drm_device *);
 extern void nv50_instmem_resume(struct drm_device *);
-extern int  nv50_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
+extern int  nv50_instmem_get(struct nouveau_gpuobj *, struct nouveau_channel *,
+                             u32 size, u32 align);
 extern void nv50_instmem_put(struct nouveau_gpuobj *);
 extern int  nv50_instmem_map(struct nouveau_gpuobj *);
 extern void nv50_instmem_unmap(struct nouveau_gpuobj *);
@@ -1247,10 +1270,9 @@ extern int nv04_crtc_create(struct drm_device *, int index);
 
 /* nouveau_bo.c */
 extern struct ttm_bo_driver nouveau_bo_driver;
-extern int nouveau_bo_new(struct drm_device *, struct nouveau_channel *,
-                          int size, int align, uint32_t flags,
-                          uint32_t tile_mode, uint32_t tile_flags,
-                          struct nouveau_bo **);
+extern int nouveau_bo_new(struct drm_device *, int size, int align,
+                          uint32_t flags, uint32_t tile_mode,
+                          uint32_t tile_flags, struct nouveau_bo **);
 extern int nouveau_bo_pin(struct nouveau_bo *, uint32_t flags);
 extern int nouveau_bo_unpin(struct nouveau_bo *);
 extern int nouveau_bo_map(struct nouveau_bo *);
@@ -1265,6 +1287,12 @@ extern void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *);
 extern int nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
                                bool no_wait_reserve, bool no_wait_gpu);
 
+extern struct nouveau_vma *
+nouveau_bo_vma_find(struct nouveau_bo *, struct nouveau_vm *);
+extern int  nouveau_bo_vma_add(struct nouveau_bo *, struct nouveau_vm *,
+                               struct nouveau_vma *);
+extern void nouveau_bo_vma_del(struct nouveau_bo *, struct nouveau_vma *);
+
 /* nouveau_fence.c */
 struct nouveau_fence;
 extern int nouveau_fence_init(struct drm_device *);
@@ -1310,12 +1338,14 @@ static inline struct nouveau_fence *nouveau_fence_ref(struct nouveau_fence *obj)
 }
 
 /* nouveau_gem.c */
-extern int nouveau_gem_new(struct drm_device *, struct nouveau_channel *,
-                           int size, int align, uint32_t domain,
-                           uint32_t tile_mode, uint32_t tile_flags,
-                           struct nouveau_bo **);
+extern int nouveau_gem_new(struct drm_device *, int size, int align,
+                           uint32_t domain, uint32_t tile_mode,
+                           uint32_t tile_flags, struct nouveau_bo **);
 extern int nouveau_gem_object_new(struct drm_gem_object *);
 extern void nouveau_gem_object_del(struct drm_gem_object *);
+extern int nouveau_gem_object_open(struct drm_gem_object *, struct drm_file *);
+extern void nouveau_gem_object_close(struct drm_gem_object *,
+                                     struct drm_file *);
 extern int nouveau_gem_ioctl_new(struct drm_device *, void *,
                                  struct drm_file *);
 extern int nouveau_gem_ioctl_pushbuf(struct drm_device *, void *,
diff --git a/drivers/gpu/drm/nouveau/nouveau_fb.h b/drivers/gpu/drm/nouveau/nouveau_fb.h
index a3a88ad00f8..95c843e684b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fb.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fb.h
@@ -30,6 +30,7 @@
 struct nouveau_framebuffer {
         struct drm_framebuffer base;
         struct nouveau_bo *nvbo;
+        struct nouveau_vma vma;
         u32 r_dma;
         u32 r_format;
         u32 r_pitch;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 39aee6d4daf..14a8627efe4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -279,6 +279,7 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
         struct fb_info *info;
         struct drm_framebuffer *fb;
         struct nouveau_framebuffer *nouveau_fb;
+        struct nouveau_channel *chan;
         struct nouveau_bo *nvbo;
         struct drm_mode_fb_cmd mode_cmd;
         struct pci_dev *pdev = dev->pdev;
@@ -296,8 +297,8 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
         size = mode_cmd.pitch * mode_cmd.height;
         size = roundup(size, PAGE_SIZE);
 
-        ret = nouveau_gem_new(dev, dev_priv->channel, size, 0,
-                              NOUVEAU_GEM_DOMAIN_VRAM, 0, 0x0000, &nvbo);
+        ret = nouveau_gem_new(dev, size, 0, NOUVEAU_GEM_DOMAIN_VRAM,
+                              0, 0x0000, &nvbo);
         if (ret) {
                 NV_ERROR(dev, "failed to allocate framebuffer\n");
                 goto out;
@@ -318,6 +319,15 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
                 goto out;
         }
 
+        chan = nouveau_nofbaccel ? NULL : dev_priv->channel;
+        if (chan && dev_priv->card_type >= NV_50) {
+                ret = nouveau_bo_vma_add(nvbo, chan->vm, &nfbdev->nouveau_fb.vma);
+                if (ret) {
+                        NV_ERROR(dev, "failed to map fb into chan: %d\n", ret);
+                        chan = NULL;
+                }
+        }
+
         mutex_lock(&dev->struct_mutex);
 
         info = framebuffer_alloc(0, device);
@@ -448,6 +458,7 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
 
         if (nouveau_fb->nvbo) {
                 nouveau_bo_unmap(nouveau_fb->nvbo);
+                nouveau_bo_vma_del(nouveau_fb->nvbo, &nouveau_fb->vma);
                 drm_gem_object_unreference_unlocked(nouveau_fb->nvbo->gem);
                 nouveau_fb->nvbo = NULL;
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 7347075ca5b..ae22dfaa2a0 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -336,6 +336,7 @@ semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 {
         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct nouveau_fence *fence = NULL;
+        u64 offset = chan->fence.vma.offset + sema->mem->start;
         int ret;
 
         if (dev_priv->chipset < 0x84) {
@@ -345,13 +346,10 @@ semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 
                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
                 OUT_RING  (chan, NvSema);
-                OUT_RING  (chan, sema->mem->start);
+                OUT_RING  (chan, offset);
                 OUT_RING  (chan, 1);
         } else
         if (dev_priv->chipset < 0xc0) {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
-
                 ret = RING_SPACE(chan, 7);
                 if (ret)
                         return ret;
@@ -364,9 +362,6 @@ semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
                 OUT_RING  (chan, 1);
                 OUT_RING  (chan, 1); /* ACQUIRE_EQ */
         } else {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
-
                 ret = RING_SPACE(chan, 5);
                 if (ret)
                         return ret;
@@ -394,6 +389,7 @@ semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 {
         struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct nouveau_fence *fence = NULL;
+        u64 offset = chan->fence.vma.offset + sema->mem->start;
         int ret;
 
         if (dev_priv->chipset < 0x84) {
@@ -403,14 +399,11 @@ semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 
                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
                 OUT_RING  (chan, NvSema);
-                OUT_RING  (chan, sema->mem->start);
+                OUT_RING  (chan, offset);
                 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
                 OUT_RING  (chan, 1);
         } else
         if (dev_priv->chipset < 0xc0) {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
-
                 ret = RING_SPACE(chan, 7);
                 if (ret)
                         return ret;
@@ -423,9 +416,6 @@ semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
                 OUT_RING  (chan, 1);
                 OUT_RING  (chan, 2); /* RELEASE */
         } else {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
-
                 ret = RING_SPACE(chan, 5);
                 if (ret)
                         return ret;
@@ -540,10 +530,15 @@ nouveau_fence_channel_init(struct nouveau_channel *chan)
                 nouveau_gpuobj_ref(NULL, &obj);
                 if (ret)
                         return ret;
+        } else
+        if (USE_SEMA(dev)) {
+                /* map fence bo into channel's vm */
+                ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
+                                         &chan->fence.vma);
+                if (ret)
+                        return ret;
         }
 
-        INIT_LIST_HEAD(&chan->fence.pending);
-        spin_lock_init(&chan->fence.lock);
         atomic_set(&chan->fence.last_sequence_irq, 0);
         return 0;
 }
@@ -551,10 +546,10 @@ nouveau_fence_channel_init(struct nouveau_channel *chan)
 void
 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 {
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct nouveau_fence *tmp, *fence;
 
         spin_lock(&chan->fence.lock);
-
         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
                 fence->signalled = true;
                 list_del(&fence->entry);
@@ -564,8 +559,9 @@ nouveau_fence_channel_fini(struct nouveau_channel *chan)
 
                 kref_put(&fence->refcount, nouveau_fence_del);
         }
-
         spin_unlock(&chan->fence.lock);
+
+        nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
 }
 
 int
@@ -577,7 +573,7 @@ nouveau_fence_init(struct drm_device *dev)
 
         /* Create a shared VRAM heap for cross-channel sync. */
         if (USE_SEMA(dev)) {
-                ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM,
+                ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
                                      0, 0, &dev_priv->fence.bo);
                 if (ret)
                         return ret;
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index b52e4601824..5f0bc57fdaa 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -60,9 +60,71 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
 }
 
 int
-nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
-                int size, int align, uint32_t domain, uint32_t tile_mode,
-                uint32_t tile_flags, struct nouveau_bo **pnvbo)
+nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
+{
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+        struct nouveau_bo *nvbo = nouveau_gem_object(gem);
+        struct nouveau_vma *vma;
+        int ret;
+
+        if (!fpriv->vm)
+                return 0;
+
+        ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
+        if (ret)
+                return ret;
+
+        vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+        if (!vma) {
+                vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+                if (!vma) {
+                        ret = -ENOMEM;
+                        goto out;
+                }
+
+                ret = nouveau_bo_vma_add(nvbo, fpriv->vm, vma);
+                if (ret) {
+                        kfree(vma);
+                        goto out;
+                }
+        } else {
+                vma->refcount++;
+        }
+
+out:
+        ttm_bo_unreserve(&nvbo->bo);
+        return ret;
+}
+
+void
+nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
+{
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+        struct nouveau_bo *nvbo = nouveau_gem_object(gem);
+        struct nouveau_vma *vma;
+        int ret;
+
+        if (!fpriv->vm)
+                return;
+
+        ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
+        if (ret)
+                return;
+
+        vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+        if (vma) {
+                if (--vma->refcount == 0) {
+                        nouveau_bo_vma_del(nvbo, vma);
+                        kfree(vma);
+                }
+        }
+        ttm_bo_unreserve(&nvbo->bo);
+}
+
+int
+nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
+                uint32_t tile_mode, uint32_t tile_flags,
+                struct nouveau_bo **pnvbo)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_bo *nvbo;
@@ -76,7 +138,7 @@ nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
         if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
                 flags |= TTM_PL_FLAG_SYSTEM;
 
-        ret = nouveau_bo_new(dev, chan, size, align, flags, tile_mode,
+        ret = nouveau_bo_new(dev, size, align, flags, tile_mode,
                              tile_flags, pnvbo);
         if (ret)
                 return ret;
@@ -103,17 +165,28 @@ nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
 }
 
 static int
-nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep)
+nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
+                 struct drm_nouveau_gem_info *rep)
 {
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
         struct nouveau_bo *nvbo = nouveau_gem_object(gem);
+        struct nouveau_vma *vma;
 
         if (nvbo->bo.mem.mem_type == TTM_PL_TT)
                 rep->domain = NOUVEAU_GEM_DOMAIN_GART;
         else
                 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
 
-        rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
         rep->offset = nvbo->bo.offset;
+        if (fpriv->vm) {
+                vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+                if (!vma)
+                        return -EINVAL;
+
+                rep->offset = vma->offset;
+        }
+
+        rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
         rep->map_handle = nvbo->bo.addr_space_offset;
         rep->tile_mode = nvbo->tile_mode;
         rep->tile_flags = nvbo->tile_flags;
@@ -127,7 +200,6 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct drm_nouveau_gem_new *req = data;
         struct nouveau_bo *nvbo = NULL;
-        struct nouveau_channel *chan = NULL;
         int ret = 0;
 
         if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL))
@@ -138,28 +210,21 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
                 return -EINVAL;
         }
 
-        if (req->channel_hint) {
-                chan = nouveau_channel_get(dev, file_priv, req->channel_hint);
-                if (IS_ERR(chan))
-                        return PTR_ERR(chan);
-        }
-
-        ret = nouveau_gem_new(dev, chan, req->info.size, req->align,
+        ret = nouveau_gem_new(dev, req->info.size, req->align,
                               req->info.domain, req->info.tile_mode,
                               req->info.tile_flags, &nvbo);
-        if (chan)
-                nouveau_channel_put(&chan);
         if (ret)
                 return ret;
 
-        ret = nouveau_gem_info(nvbo->gem, &req->info);
-        if (ret)
-                goto out;
-
         ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle);
+        if (ret == 0) {
+                ret = nouveau_gem_info(file_priv, nvbo->gem, &req->info);
+                if (ret)
+                        drm_gem_handle_delete(file_priv, req->info.handle);
+        }
+
         /* drop reference from allocate - handle holds it now */
         drm_gem_object_unreference_unlocked(nvbo->gem);
-out:
         return ret;
 }
 
@@ -318,6 +383,7 @@ static int
 validate_list(struct nouveau_channel *chan, struct list_head *list,
               struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr)
 {
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
         struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
                                 (void __force __user *)(uintptr_t)user_pbbo_ptr;
         struct drm_device *dev = chan->dev;
@@ -356,24 +422,26 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
                         return ret;
                 }
 
-                if (nvbo->bo.offset == b->presumed.offset &&
-                    ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
-                      b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
-                     (nvbo->bo.mem.mem_type == TTM_PL_TT &&
-                      b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
-                        continue;
+                if (dev_priv->card_type < NV_50) {
+                        if (nvbo->bo.offset == b->presumed.offset &&
+                            ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
+                              b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
+                             (nvbo->bo.mem.mem_type == TTM_PL_TT &&
+                              b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
+                                continue;
 
-                if (nvbo->bo.mem.mem_type == TTM_PL_TT)
-                        b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
-                else
-                        b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
-                b->presumed.offset = nvbo->bo.offset;
-                b->presumed.valid = 0;
-                relocs++;
-
-                if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed,
-                                     &b->presumed, sizeof(b->presumed)))
-                        return -EFAULT;
+                        if (nvbo->bo.mem.mem_type == TTM_PL_TT)
+                                b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
+                        else
+                                b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
+                        b->presumed.offset = nvbo->bo.offset;
+                        b->presumed.valid = 0;
+                        relocs++;
+
+                        if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed,
+                                             &b->presumed, sizeof(b->presumed)))
+                                return -EFAULT;
+                }
         }
 
         return relocs;
@@ -548,7 +616,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
         struct nouveau_fence *fence = NULL;
         int i, j, ret = 0, do_reloc = 0;
 
-        chan = nouveau_channel_get(dev, file_priv, req->channel);
+        chan = nouveau_channel_get(file_priv, req->channel);
         if (IS_ERR(chan))
                 return PTR_ERR(chan);
 
@@ -782,7 +850,7 @@ nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
         if (!gem)
                 return -ENOENT;
 
-        ret = nouveau_gem_info(gem, req);
+        ret = nouveau_gem_info(file_priv, gem, req);
         drm_gem_object_unreference_unlocked(gem);
         return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_irq.c b/drivers/gpu/drm/nouveau/nouveau_irq.c
index 2ba7265bc96..868c7fd7485 100644
--- a/drivers/gpu/drm/nouveau/nouveau_irq.c
+++ b/drivers/gpu/drm/nouveau/nouveau_irq.c
@@ -79,7 +79,7 @@ nouveau_irq_handler(DRM_IRQ_ARGS)
         int i;
 
         stat = nv_rd32(dev, NV03_PMC_INTR_0);
-        if (!stat)
+        if (stat == 0 || stat == ~0)
                 return IRQ_NONE;
 
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 5ee14d216ce..f9ae2fc3d6f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -397,7 +397,7 @@ nouveau_mem_vram_init(struct drm_device *dev)
                 if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(40)))
                         dma_bits = 40;
         } else
-        if (0 && drm_pci_device_is_pcie(dev) &&
+        if (0 && pci_is_pcie(dev->pdev) &&
             dev_priv->chipset  > 0x40 &&
             dev_priv->chipset != 0x45) {
                 if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(39)))
@@ -423,38 +423,6 @@ nouveau_mem_vram_init(struct drm_device *dev)
                 return ret;
         }
 
-        /* reserve space at end of VRAM for PRAMIN */
-        if (dev_priv->card_type >= NV_50) {
-                dev_priv->ramin_rsvd_vram = 1 * 1024 * 1024;
-        } else
-        if (dev_priv->card_type >= NV_40) {
-                u32 vs = hweight8((nv_rd32(dev, 0x001540) & 0x0000ff00) >> 8);
-                u32 rsvd;
-
-                /* estimate grctx size, the magics come from nv40_grctx.c */
-                if      (dev_priv->chipset == 0x40) rsvd = 0x6aa0 * vs;
-                else if (dev_priv->chipset  < 0x43) rsvd = 0x4f00 * vs;
-                else if (nv44_graph_class(dev))     rsvd = 0x4980 * vs;
-                else                                rsvd = 0x4a40 * vs;
-                rsvd += 16 * 1024;
-                rsvd *= dev_priv->engine.fifo.channels;
-
-                /* pciegart table */
-                if (drm_pci_device_is_pcie(dev))
-                        rsvd += 512 * 1024;
-
-                /* object storage */
-                rsvd += 512 * 1024;
-
-                dev_priv->ramin_rsvd_vram = round_up(rsvd, 4096);
-        } else {
-                dev_priv->ramin_rsvd_vram = 512 * 1024;
-        }
-
-        ret = dev_priv->engine.vram.init(dev);
-        if (ret)
-                return ret;
-
         NV_INFO(dev, "Detected %dMiB VRAM\n", (int)(dev_priv->vram_size >> 20));
         if (dev_priv->vram_sys_base) {
                 NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
@@ -479,7 +447,7 @@ nouveau_mem_vram_init(struct drm_device *dev)
         }
 
         if (dev_priv->card_type < NV_50) {
-                ret = nouveau_bo_new(dev, NULL, 256*1024, 0, TTM_PL_FLAG_VRAM,
+                ret = nouveau_bo_new(dev, 256*1024, 0, TTM_PL_FLAG_VRAM,
                                      0, 0, &dev_priv->vga_ram);
                 if (ret == 0)
                         ret = nouveau_bo_pin(dev_priv->vga_ram,
@@ -729,37 +697,31 @@ nouveau_mem_timing_fini(struct drm_device *dev)
 }
 
 static int
-nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long p_size)
+nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
 {
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
-        struct nouveau_mm *mm;
-        u64 size, block, rsvd;
-        int ret;
-
-        rsvd  = (256 * 1024); /* vga memory */
-        size  = (p_size << PAGE_SHIFT) - rsvd;
-        block = dev_priv->vram_rblock_size;
-
-        ret = nouveau_mm_init(&mm, rsvd >> 12, size >> 12, block >> 12);
-        if (ret)
-                return ret;
-
-        man->priv = mm;
+        /* nothing to do */
         return 0;
 }
 
 static int
 nouveau_vram_manager_fini(struct ttm_mem_type_manager *man)
 {
-        struct nouveau_mm *mm = man->priv;
-        int ret;
+        /* nothing to do */
+        return 0;
+}
 
-        ret = nouveau_mm_fini(&mm);
-        if (ret)
-                return ret;
+static inline void
+nouveau_mem_node_cleanup(struct nouveau_mem *node)
+{
+        if (node->vma[0].node) {
+                nouveau_vm_unmap(&node->vma[0]);
+                nouveau_vm_put(&node->vma[0]);
+        }
 
-        man->priv = NULL;
-        return 0;
+        if (node->vma[1].node) {
+                nouveau_vm_unmap(&node->vma[1]);
+                nouveau_vm_put(&node->vma[1]);
+        }
 }
 
 static void
@@ -768,14 +730,9 @@ nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
 {
         struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
         struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-        struct nouveau_mem *node = mem->mm_node;
         struct drm_device *dev = dev_priv->dev;
 
-        if (node->tmp_vma.node) {
-                nouveau_vm_unmap(&node->tmp_vma);
-                nouveau_vm_put(&node->tmp_vma);
-        }
-
+        nouveau_mem_node_cleanup(mem->mm_node);
         vram->put(dev, (struct nouveau_mem **)&mem->mm_node);
 }
 
@@ -794,7 +751,7 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
         int ret;
 
         if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)
-                size_nc = 1 << nvbo->vma.node->type;
+                size_nc = 1 << nvbo->page_shift;
 
         ret = vram->get(dev, mem->num_pages << PAGE_SHIFT,
                         mem->page_alignment << PAGE_SHIFT, size_nc,
@@ -804,9 +761,7 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
                 return (ret == -ENOSPC) ? 0 : ret;
         }
 
-        node->page_shift = 12;
-        if (nvbo->vma.node)
-                node->page_shift = nvbo->vma.node->type;
+        node->page_shift = nvbo->page_shift;
 
         mem->mm_node = node;
         mem->start   = node->offset >> PAGE_SHIFT;
@@ -862,15 +817,9 @@ static void
 nouveau_gart_manager_del(struct ttm_mem_type_manager *man,
                          struct ttm_mem_reg *mem)
 {
-        struct nouveau_mem *node = mem->mm_node;
-
-        if (node->tmp_vma.node) {
-                nouveau_vm_unmap(&node->tmp_vma);
-                nouveau_vm_put(&node->tmp_vma);
-        }
-
+        nouveau_mem_node_cleanup(mem->mm_node);
+        kfree(mem->mm_node);
         mem->mm_node = NULL;
-        kfree(node);
 }
 
 static int
@@ -880,11 +829,7 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
                          struct ttm_mem_reg *mem)
 {
         struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-        struct nouveau_bo *nvbo = nouveau_bo(bo);
-        struct nouveau_vma *vma = &nvbo->vma;
-        struct nouveau_vm *vm = vma->vm;
         struct nouveau_mem *node;
-        int ret;
 
         if (unlikely((mem->num_pages << PAGE_SHIFT) >=
                      dev_priv->gart_info.aper_size))
@@ -893,24 +838,8 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
         node = kzalloc(sizeof(*node), GFP_KERNEL);
         if (!node)
                 return -ENOMEM;
+        node->page_shift = 12;
 
-        /* This node must be for evicting large-paged VRAM
-         * to system memory.  Due to a nv50 limitation of
-         * not being able to mix large/small pages within
-         * the same PDE, we need to create a temporary
-         * small-paged VMA for the eviction.
-         */
-        if (vma->node->type != vm->spg_shift) {
-                ret = nouveau_vm_get(vm, (u64)vma->node->length << 12,
-                                     vm->spg_shift, NV_MEM_ACCESS_RW,
-                                     &node->tmp_vma);
-                if (ret) {
-                        kfree(node);
-                        return ret;
-                }
-        }
-
-        node->page_shift = nvbo->vma.node->type;
         mem->mm_node = node;
         mem->start   = 0;
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.c b/drivers/gpu/drm/nouveau/nouveau_mm.c
index 7609756b6fa..1640dec3b82 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.c
@@ -158,11 +158,18 @@ int
 nouveau_mm_fini(struct nouveau_mm **prmm)
 {
         struct nouveau_mm *rmm = *prmm;
-        struct nouveau_mm_node *heap =
+        struct nouveau_mm_node *node, *heap =
                 list_first_entry(&rmm->nodes, struct nouveau_mm_node, nl_entry);
 
-        if (!list_is_singular(&rmm->nodes))
+        if (!list_is_singular(&rmm->nodes)) {
+                printk(KERN_ERR "nouveau_mm not empty at destroy time!\n");
+                list_for_each_entry(node, &rmm->nodes, nl_entry) {
+                        printk(KERN_ERR "0x%02x: 0x%08x 0x%08x\n",
+                               node->type, node->offset, node->length);
+                }
+                WARN_ON(1);
                 return -EBUSY;
+        }
 
         kfree(heap);
         kfree(rmm);
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.h b/drivers/gpu/drm/nouveau/nouveau_mm.h
index 1f7483aae9a..b9c016d2155 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.h
@@ -52,6 +52,7 @@ int  nouveau_mm_get(struct nouveau_mm *, int type, u32 size, u32 size_nc,
 void nouveau_mm_put(struct nouveau_mm *, struct nouveau_mm_node *);
 
 int  nv50_vram_init(struct drm_device *);
+void nv50_vram_fini(struct drm_device *);
 int  nv50_vram_new(struct drm_device *, u64 size, u32 align, u32 size_nc,
                     u32 memtype, struct nouveau_mem **);
 void nv50_vram_del(struct drm_device *, struct nouveau_mem **);
diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c
index 5b39718ae1f..6abdbe6530a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_notifier.c
+++ b/drivers/gpu/drm/nouveau/nouveau_notifier.c
@@ -34,6 +34,7 @@ int
 nouveau_notifier_init_channel(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_bo *ntfy = NULL;
         uint32_t flags, ttmpl;
         int ret;
@@ -46,7 +47,7 @@ nouveau_notifier_init_channel(struct nouveau_channel *chan)
                 ttmpl = TTM_PL_FLAG_TT;
         }
 
-        ret = nouveau_gem_new(dev, NULL, PAGE_SIZE, 0, flags, 0, 0, &ntfy);
+        ret = nouveau_gem_new(dev, PAGE_SIZE, 0, flags, 0, 0, &ntfy);
         if (ret)
                 return ret;
 
@@ -58,14 +59,22 @@ nouveau_notifier_init_channel(struct nouveau_channel *chan)
         if (ret)
                 goto out_err;
 
+        if (dev_priv->card_type >= NV_50) {
+                ret = nouveau_bo_vma_add(ntfy, chan->vm, &chan->notifier_vma);
+                if (ret)
+                        goto out_err;
+        }
+
         ret = drm_mm_init(&chan->notifier_heap, 0, ntfy->bo.mem.size);
         if (ret)
                 goto out_err;
 
         chan->notifier_bo = ntfy;
 out_err:
-        if (ret)
+        if (ret) {
+                nouveau_bo_vma_del(ntfy, &chan->notifier_vma);
                 drm_gem_object_unreference_unlocked(ntfy->gem);
+        }
 
         return ret;
 }
@@ -78,6 +87,7 @@ nouveau_notifier_takedown_channel(struct nouveau_channel *chan)
         if (!chan->notifier_bo)
                 return;
 
+        nouveau_bo_vma_del(chan->notifier_bo, &chan->notifier_vma);
         nouveau_bo_unmap(chan->notifier_bo);
         mutex_lock(&dev->struct_mutex);
         nouveau_bo_unpin(chan->notifier_bo);
@@ -122,10 +132,10 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
                         target = NV_MEM_TARGET_VRAM;
                 else
                         target = NV_MEM_TARGET_GART;
-                offset  = chan->notifier_bo->bo.mem.start << PAGE_SHIFT;
+                offset  = chan->notifier_bo->bo.offset;
         } else {
                 target = NV_MEM_TARGET_VM;
-                offset = chan->notifier_bo->vma.offset;
+                offset = chan->notifier_vma.offset;
         }
         offset += mem->start;
 
@@ -183,7 +193,7 @@ nouveau_ioctl_notifier_alloc(struct drm_device *dev, void *data,
         if (unlikely(dev_priv->card_type >= NV_C0))
                 return -EINVAL;
 
-        chan = nouveau_channel_get(dev, file_priv, na->channel);
+        chan = nouveau_channel_get(file_priv, na->channel);
         if (IS_ERR(chan))
                 return PTR_ERR(chan);
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_object.c b/drivers/gpu/drm/nouveau/nouveau_object.c
index 8f97016f5b2..159b7c437d3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_object.c
+++ b/drivers/gpu/drm/nouveau/nouveau_object.c
@@ -125,7 +125,7 @@ nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
         int ret = -EINVAL;
 
         spin_lock_irqsave(&dev_priv->channels.lock, flags);
-        if (chid > 0 && chid < dev_priv->engine.fifo.channels)
+        if (chid >= 0 && chid < dev_priv->engine.fifo.channels)
                 chan = dev_priv->channels.ptr[chid];
         if (chan)
                 ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
@@ -191,7 +191,7 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
         list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
         spin_unlock(&dev_priv->ramin_lock);
 
-        if (chan) {
+        if (!(flags & NVOBJ_FLAG_VM) && chan) {
                 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
                 if (ramin)
                         ramin = drm_mm_get_block(ramin, size, align);
@@ -208,7 +208,7 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
                 gpuobj->vinst = ramin->start + chan->ramin->vinst;
                 gpuobj->node  = ramin;
         } else {
-                ret = instmem->get(gpuobj, size, align);
+                ret = instmem->get(gpuobj, chan, size, align);
                 if (ret) {
                         nouveau_gpuobj_ref(NULL, &gpuobj);
                         return ret;
@@ -690,35 +690,64 @@ nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
         return 0;
 }
 
+static int
+nvc0_gpuobj_channel_init(struct nouveau_channel *chan, struct nouveau_vm *vm)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_gpuobj *pgd = NULL;
+        struct nouveau_vm_pgd *vpgd;
+        int ret, i;
+
+        ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0, &chan->ramin);
+        if (ret)
+                return ret;
+
+        /* create page directory for this vm if none currently exists,
+         * will be destroyed automagically when last reference to the
+         * vm is removed
+         */
+        if (list_empty(&vm->pgd_list)) {
+                ret = nouveau_gpuobj_new(dev, NULL, 65536, 0x1000, 0, &pgd);
+                if (ret)
+                        return ret;
+        }
+        nouveau_vm_ref(vm, &chan->vm, pgd);
+        nouveau_gpuobj_ref(NULL, &pgd);
+
+        /* point channel at vm's page directory */
+        vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
+        nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
+        nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
+        nv_wo32(chan->ramin, 0x0208, 0xffffffff);
+        nv_wo32(chan->ramin, 0x020c, 0x000000ff);
+
+        /* map display semaphore buffers into channel's vm */
+        for (i = 0; i < 2; i++) {
+                struct nv50_display_crtc *dispc = &nv50_display(dev)->crtc[i];
+
+                ret = nouveau_bo_vma_add(dispc->sem.bo, chan->vm,
+                                         &chan->dispc_vma[i]);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
 int
 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                             uint32_t vram_h, uint32_t tt_h)
 {
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(chan->file_priv);
+        struct nouveau_vm *vm = fpriv ? fpriv->vm : dev_priv->chan_vm;
         struct nouveau_gpuobj *vram = NULL, *tt = NULL;
         int ret, i;
 
         NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
-
-        if (dev_priv->card_type == NV_C0) {
-                struct nouveau_vm *vm = dev_priv->chan_vm;
-                struct nouveau_vm_pgd *vpgd;
-
-                ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0,
-                                         &chan->ramin);
-                if (ret)
-                        return ret;
-
-                nouveau_vm_ref(vm, &chan->vm, NULL);
-
-                vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
-                nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
-                nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
-                nv_wo32(chan->ramin, 0x0208, 0xffffffff);
-                nv_wo32(chan->ramin, 0x020c, 0x000000ff);
-                return 0;
-        }
+        if (dev_priv->card_type == NV_C0)
+                return nvc0_gpuobj_channel_init(chan, vm);
 
         /* Allocate a chunk of memory for per-channel object storage */
         ret = nouveau_gpuobj_channel_init_pramin(chan);
@@ -731,7 +760,7 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
          *  - Allocate per-channel page-directory
          *  - Link with shared channel VM
          */
-        if (dev_priv->chan_vm) {
+        if (vm) {
                 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
                 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
                 u32 vm_pinst = chan->ramin->pinst;
@@ -744,7 +773,7 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                 if (ret)
                         return ret;
 
-                nouveau_vm_ref(dev_priv->chan_vm, &chan->vm, chan->vm_pd);
+                nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
         }
 
         /* RAMHT */
@@ -768,7 +797,7 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                         struct nouveau_gpuobj *sem = NULL;
                         struct nv50_display_crtc *dispc =
                                 &nv50_display(dev)->crtc[i];
-                        u64 offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
+                        u64 offset = dispc->sem.bo->bo.offset;
 
                         ret = nouveau_gpuobj_dma_new(chan, 0x3d, offset, 0xfff,
                                                      NV_MEM_ACCESS_RW,
@@ -841,13 +870,22 @@ void
 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 {
         struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int i;
 
         NV_DEBUG(dev, "ch%d\n", chan->id);
 
-        nouveau_ramht_ref(NULL, &chan->ramht, chan);
+        if (dev_priv->card_type >= NV_50) {
+                struct nv50_display *disp = nv50_display(dev);
+
+                for (i = 0; i < 2; i++) {
+                        struct nv50_display_crtc *dispc = &disp->crtc[i];
+                        nouveau_bo_vma_del(dispc->sem.bo, &chan->dispc_vma[i]);
+                }
 
-        nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
-        nouveau_gpuobj_ref(NULL, &chan->vm_pd);
+                nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
+                nouveau_gpuobj_ref(NULL, &chan->vm_pd);
+        }
 
         if (drm_mm_initialized(&chan->ramin_heap))
                 drm_mm_takedown(&chan->ramin_heap);
@@ -909,7 +947,7 @@ int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
         if (init->handle == ~0)
                 return -EINVAL;
 
-        chan = nouveau_channel_get(dev, file_priv, init->channel);
+        chan = nouveau_channel_get(file_priv, init->channel);
         if (IS_ERR(chan))
                 return PTR_ERR(chan);
 
@@ -936,7 +974,7 @@ int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
         struct nouveau_channel *chan;
         int ret;
 
-        chan = nouveau_channel_get(dev, file_priv, objfree->channel);
+        chan = nouveau_channel_get(file_priv, objfree->channel);
         if (IS_ERR(chan))
                 return PTR_ERR(chan);
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index 82fad914e64..2706cb3d871 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -37,8 +37,11 @@ nouveau_sgdma_populate(struct ttm_backend *be, unsigned long num_pages,
                 return -ENOMEM;
 
         nvbe->ttm_alloced = kmalloc(sizeof(bool) * num_pages, GFP_KERNEL);
-        if (!nvbe->ttm_alloced)
+        if (!nvbe->ttm_alloced) {
+                kfree(nvbe->pages);
+                nvbe->pages = NULL;
                 return -ENOMEM;
+        }
 
         nvbe->nr_pages = 0;
         while (num_pages--) {
@@ -126,7 +129,7 @@ nv04_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
 
                 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
                         nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
-                        dma_offset += NV_CTXDMA_PAGE_SIZE;
+                        offset_l += NV_CTXDMA_PAGE_SIZE;
                 }
         }
 
@@ -429,7 +432,7 @@ nouveau_sgdma_init(struct drm_device *dev)
         u32 aper_size, align;
         int ret;
 
-        if (dev_priv->card_type >= NV_40 && drm_pci_device_is_pcie(dev))
+        if (dev_priv->card_type >= NV_40 && pci_is_pcie(dev->pdev))
                 aper_size = 512 * 1024 * 1024;
         else
                 aper_size = 64 * 1024 * 1024;
@@ -458,7 +461,7 @@ nouveau_sgdma_init(struct drm_device *dev)
                 dev_priv->gart_info.type = NOUVEAU_GART_HW;
                 dev_priv->gart_info.func = &nv50_sgdma_backend;
         } else
-        if (0 && drm_pci_device_is_pcie(dev) &&
+        if (0 && pci_is_pcie(dev->pdev) &&
             dev_priv->chipset > 0x40 && dev_priv->chipset != 0x45) {
                 if (nv44_graph_class(dev)) {
                         dev_priv->gart_info.func = &nv44_sgdma_backend;
diff --git a/drivers/gpu/drm/nouveau/nouveau_state.c b/drivers/gpu/drm/nouveau/nouveau_state.c
index 731acea865b..10656e430b4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_state.c
+++ b/drivers/gpu/drm/nouveau/nouveau_state.c
@@ -91,6 +91,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->pm.clock_pre            = nv04_pm_clock_pre;
                 engine->pm.clock_set            = nv04_pm_clock_set;
                 engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                 engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x10:
@@ -139,6 +140,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->pm.clock_pre            = nv04_pm_clock_pre;
                 engine->pm.clock_set            = nv04_pm_clock_set;
                 engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                 engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x20:
@@ -187,6 +189,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->pm.clock_pre            = nv04_pm_clock_pre;
                 engine->pm.clock_set            = nv04_pm_clock_set;
                 engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                 engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x30:
@@ -237,6 +240,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->pm.voltage_get          = nouveau_voltage_gpio_get;
                 engine->pm.voltage_set          = nouveau_voltage_gpio_set;
                 engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                 engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x40:
@@ -289,6 +293,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->pm.voltage_set          = nouveau_voltage_gpio_set;
                 engine->pm.temp_get             = nv40_temp_get;
                 engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                 engine->vram.flags_valid        = nouveau_mem_flags_valid;
                 break;
         case 0x50:
@@ -366,6 +371,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 else
                         engine->pm.temp_get     = nv40_temp_get;
                 engine->vram.init               = nv50_vram_init;
+                engine->vram.takedown           = nv50_vram_fini;
                 engine->vram.get                = nv50_vram_new;
                 engine->vram.put                = nv50_vram_del;
                 engine->vram.flags_valid        = nv50_vram_flags_valid;
@@ -411,9 +417,11 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                 engine->gpio.irq_unregister     = nv50_gpio_irq_unregister;
                 engine->gpio.irq_enable         = nv50_gpio_irq_enable;
                 engine->vram.init               = nvc0_vram_init;
+                engine->vram.takedown           = nv50_vram_fini;
                 engine->vram.get                = nvc0_vram_new;
                 engine->vram.put                = nv50_vram_del;
                 engine->vram.flags_valid        = nvc0_vram_flags_valid;
+                engine->pm.temp_get             = nv84_temp_get;
                 break;
         default:
                 NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset);
@@ -447,8 +455,8 @@ nouveau_card_init_channel(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         int ret;
 
-        ret = nouveau_channel_alloc(dev, &dev_priv->channel,
-                                    (struct drm_file *)-2, NvDmaFB, NvDmaTT);
+        ret = nouveau_channel_alloc(dev, &dev_priv->channel, NULL,
+                                    NvDmaFB, NvDmaTT);
         if (ret)
                 return ret;
 
@@ -527,7 +535,7 @@ nouveau_card_init(struct drm_device *dev)
 
         nouveau_pm_init(dev);
 
-        ret = nouveau_mem_vram_init(dev);
+        ret = engine->vram.init(dev);
         if (ret)
                 goto out_bios;
 
@@ -539,10 +547,14 @@ nouveau_card_init(struct drm_device *dev)
         if (ret)
                 goto out_gpuobj;
 
-        ret = nouveau_mem_gart_init(dev);
+        ret = nouveau_mem_vram_init(dev);
         if (ret)
                 goto out_instmem;
 
+        ret = nouveau_mem_gart_init(dev);
+        if (ret)
+                goto out_ttmvram;
+
         /* PMC */
         ret = engine->mc.init(dev);
         if (ret)
@@ -563,7 +575,7 @@ nouveau_card_init(struct drm_device *dev)
         if (ret)
                 goto out_timer;
 
-        if (!nouveau_noaccel) {
+        if (!dev_priv->noaccel) {
                 switch (dev_priv->card_type) {
                 case NV_04:
                         nv04_graph_create(dev);
@@ -675,14 +687,14 @@ out_vblank:
         drm_vblank_cleanup(dev);
         engine->display.destroy(dev);
 out_fifo:
-        if (!nouveau_noaccel)
+        if (!dev_priv->noaccel)
                 engine->fifo.takedown(dev);
 out_engine:
-        if (!nouveau_noaccel) {
+        if (!dev_priv->noaccel) {
                 for (e = e - 1; e >= 0; e--) {
                         if (!dev_priv->eng[e])
                                 continue;
-                        dev_priv->eng[e]->fini(dev, e);
+                        dev_priv->eng[e]->fini(dev, e, false);
                         dev_priv->eng[e]->destroy(dev,e );
                 }
         }
@@ -696,12 +708,14 @@ out_mc:
         engine->mc.takedown(dev);
 out_gart:
         nouveau_mem_gart_fini(dev);
+out_ttmvram:
+        nouveau_mem_vram_fini(dev);
 out_instmem:
         engine->instmem.takedown(dev);
 out_gpuobj:
         nouveau_gpuobj_takedown(dev);
 out_vram:
-        nouveau_mem_vram_fini(dev);
+        engine->vram.takedown(dev);
 out_bios:
         nouveau_pm_fini(dev);
         nouveau_bios_takedown(dev);
@@ -718,16 +732,21 @@ static void nouveau_card_takedown(struct drm_device *dev)
         struct nouveau_engine *engine = &dev_priv->engine;
         int e;
 
+        drm_kms_helper_poll_fini(dev);
+        nouveau_fbcon_fini(dev);
+
         if (dev_priv->channel) {
-                nouveau_fence_fini(dev);
                 nouveau_channel_put_unlocked(&dev_priv->channel);
+                nouveau_fence_fini(dev);
         }
 
-        if (!nouveau_noaccel) {
+        engine->display.destroy(dev);
+
+        if (!dev_priv->noaccel) {
                 engine->fifo.takedown(dev);
                 for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
                         if (dev_priv->eng[e]) {
-                                dev_priv->eng[e]->fini(dev, e);
+                                dev_priv->eng[e]->fini(dev, e, false);
                                 dev_priv->eng[e]->destroy(dev,e );
                         }
                 }
@@ -748,10 +767,11 @@ static void nouveau_card_takedown(struct drm_device *dev)
         ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_TT);
         mutex_unlock(&dev->struct_mutex);
         nouveau_mem_gart_fini(dev);
+        nouveau_mem_vram_fini(dev);
 
         engine->instmem.takedown(dev);
         nouveau_gpuobj_takedown(dev);
-        nouveau_mem_vram_fini(dev);
+        engine->vram.takedown(dev);
 
         nouveau_irq_fini(dev);
         drm_vblank_cleanup(dev);
@@ -762,6 +782,41 @@ static void nouveau_card_takedown(struct drm_device *dev)
         vga_client_register(dev->pdev, NULL, NULL, NULL);
 }
 
+int
+nouveau_open(struct drm_device *dev, struct drm_file *file_priv)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fpriv *fpriv;
+        int ret;
+
+        fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
+        if (unlikely(!fpriv))
+                return -ENOMEM;
+
+        spin_lock_init(&fpriv->lock);
+        INIT_LIST_HEAD(&fpriv->channels);
+
+        if (dev_priv->card_type == NV_50) {
+                ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
+                                     &fpriv->vm);
+                if (ret) {
+                        kfree(fpriv);
+                        return ret;
+                }
+        } else
+        if (dev_priv->card_type >= NV_C0) {
+                ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL,
+                                     &fpriv->vm);
+                if (ret) {
+                        kfree(fpriv);
+                        return ret;
+                }
+        }
+
+        file_priv->driver_priv = fpriv;
+        return 0;
+}
+
 /* here a client dies, release the stuff that was allocated for its
  * file_priv */
 void nouveau_preclose(struct drm_device *dev, struct drm_file *file_priv)
@@ -769,6 +824,14 @@ void nouveau_preclose(struct drm_device *dev, struct drm_file *file_priv)
         nouveau_channel_cleanup(dev, file_priv);
 }
 
+void
+nouveau_postclose(struct drm_device *dev, struct drm_file *file_priv)
+{
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+        nouveau_vm_ref(NULL, &fpriv->vm, NULL);
+        kfree(fpriv);
+}
+
 /* first module load, setup the mmio/fb mapping */
 /* KMS: we need mmio at load time, not when the first drm client opens. */
 int nouveau_firstopen(struct drm_device *dev)
@@ -933,6 +996,25 @@ int nouveau_load(struct drm_device *dev, unsigned long flags)
         NV_INFO(dev, "Detected an NV%2x generation card (0x%08x)\n",
                 dev_priv->card_type, reg0);
 
+        /* Determine whether we'll attempt acceleration or not, some
+         * cards are disabled by default here due to them being known
+         * non-functional, or never been tested due to lack of hw.
+         */
+        dev_priv->noaccel = !!nouveau_noaccel;
+        if (nouveau_noaccel == -1) {
+                switch (dev_priv->chipset) {
+                case 0xc1: /* known broken */
+                case 0xc8: /* never tested */
+                        NV_INFO(dev, "acceleration disabled by default, pass "
+                                     "noaccel=0 to force enable\n");
+                        dev_priv->noaccel = true;
+                        break;
+                default:
+                        dev_priv->noaccel = false;
+                        break;
+                }
+        }
+
         ret = nouveau_remove_conflicting_drivers(dev);
         if (ret)
                 goto err_mmio;
@@ -997,11 +1079,7 @@ void nouveau_lastclose(struct drm_device *dev)
 int nouveau_unload(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
 
-        drm_kms_helper_poll_fini(dev);
-        nouveau_fbcon_fini(dev);
-        engine->display.destroy(dev);
         nouveau_card_takedown(dev);
 
         iounmap(dev_priv->mmio);
@@ -1031,7 +1109,7 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
         case NOUVEAU_GETPARAM_BUS_TYPE:
                 if (drm_pci_device_is_agp(dev))
                         getparam->value = NV_AGP;
-                else if (drm_pci_device_is_pcie(dev))
+                else if (pci_is_pcie(dev->pdev))
                         getparam->value = NV_PCIE;
                 else
                         getparam->value = NV_PCI;
diff --git a/drivers/gpu/drm/nouveau/nouveau_temp.c b/drivers/gpu/drm/nouveau/nouveau_temp.c
index 649b0413b09..081ca7b03e8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_temp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_temp.c
@@ -43,7 +43,7 @@ nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
 
         /* Set the default sensor's contants */
         sensor->offset_constant = 0;
-        sensor->offset_mult = 1;
+        sensor->offset_mult = 0;
         sensor->offset_div = 1;
         sensor->slope_mult = 1;
         sensor->slope_div = 1;
@@ -99,6 +99,13 @@ nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
                         sensor->slope_mult = 431;
                         sensor->slope_div = 10000;
                         break;
+
+                case 0x67:
+                        sensor->offset_mult = -26149;
+                        sensor->offset_div = 100;
+                        sensor->slope_mult = 484;
+                        sensor->slope_div = 10000;
+                        break;
                 }
         }
 
@@ -109,7 +116,7 @@ nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
 
         /* Read the entries from the table */
         for (i = 0; i < entries; i++) {
-                u16 value = ROM16(temp[1]);
+                s16 value = ROM16(temp[1]);
 
                 switch (temp[0]) {
                 case 0x01:
@@ -160,8 +167,8 @@ nv40_sensor_setup(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
         struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
-        u32 offset = sensor->offset_mult / sensor->offset_div;
-        u32 sensor_calibration;
+        s32 offset = sensor->offset_mult / sensor->offset_div;
+        s32 sensor_calibration;
 
         /* set up the sensors */
         sensor_calibration = 120 - offset - sensor->offset_constant;
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.c b/drivers/gpu/drm/nouveau/nouveau_vm.c
index 519a6b4bba4..244fd38fdb8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.c
@@ -369,23 +369,26 @@ nouveau_vm_link(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
 }
 
 static void
-nouveau_vm_unlink(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
+nouveau_vm_unlink(struct nouveau_vm *vm, struct nouveau_gpuobj *mpgd)
 {
         struct nouveau_vm_pgd *vpgd, *tmp;
+        struct nouveau_gpuobj *pgd = NULL;
 
-        if (!pgd)
+        if (!mpgd)
                 return;
 
         mutex_lock(&vm->mm->mutex);
         list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
-                if (vpgd->obj != pgd)
-                        continue;
-
-                list_del(&vpgd->head);
-                nouveau_gpuobj_ref(NULL, &vpgd->obj);
-                kfree(vpgd);
+                if (vpgd->obj == mpgd) {
+                        pgd = vpgd->obj;
+                        list_del(&vpgd->head);
+                        kfree(vpgd);
+                        break;
+                }
         }
         mutex_unlock(&vm->mm->mutex);
+
+        nouveau_gpuobj_ref(NULL, &pgd);
 }
 
 static void
@@ -396,8 +399,8 @@ nouveau_vm_del(struct nouveau_vm *vm)
         list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
                 nouveau_vm_unlink(vm, vpgd->obj);
         }
-        WARN_ON(nouveau_mm_fini(&vm->mm) != 0);
 
+        nouveau_mm_fini(&vm->mm);
         kfree(vm->pgt);
         kfree(vm);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.h b/drivers/gpu/drm/nouveau/nouveau_vm.h
index c48a9fc2b47..579ca8cc223 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.h
@@ -41,6 +41,8 @@ struct nouveau_vm_pgd {
 };
 
 struct nouveau_vma {
+        struct list_head head;
+        int refcount;
         struct nouveau_vm *vm;
         struct nouveau_mm_node *node;
         u64 offset;
diff --git a/drivers/gpu/drm/nouveau/nv04_crtc.c b/drivers/gpu/drm/nouveau/nv04_crtc.c
index f1a3ae49199..5e45398a9e2 100644
--- a/drivers/gpu/drm/nouveau/nv04_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv04_crtc.c
@@ -781,11 +781,20 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
-        struct drm_framebuffer *drm_fb = nv_crtc->base.fb;
-        struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb);
+        struct drm_framebuffer *drm_fb;
+        struct nouveau_framebuffer *fb;
         int arb_burst, arb_lwm;
         int ret;
 
+        NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
+
+        /* no fb bound */
+        if (!atomic && !crtc->fb) {
+                NV_DEBUG_KMS(dev, "No FB bound\n");
+                return 0;
+        }
+
+
         /* If atomic, we want to switch to the fb we were passed, so
          * now we update pointers to do that.  (We don't pin; just
          * assume we're already pinned and update the base address.)
@@ -794,6 +803,8 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
                 drm_fb = passed_fb;
                 fb = nouveau_framebuffer(passed_fb);
         } else {
+                drm_fb = crtc->fb;
+                fb = nouveau_framebuffer(crtc->fb);
                 /* If not atomic, we can go ahead and pin, and unpin the
                  * old fb we were passed.
                  */
@@ -1035,7 +1046,7 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
         drm_crtc_helper_add(&nv_crtc->base, &nv04_crtc_helper_funcs);
         drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
 
-        ret = nouveau_bo_new(dev, NULL, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
+        ret = nouveau_bo_new(dev, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
                              0, 0x0000, &nv_crtc->cursor.nvbo);
         if (!ret) {
                 ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
diff --git a/drivers/gpu/drm/nouveau/nv04_graph.c b/drivers/gpu/drm/nouveau/nv04_graph.c
index 3626ee7db3b..dbdea8ed392 100644
--- a/drivers/gpu/drm/nouveau/nv04_graph.c
+++ b/drivers/gpu/drm/nouveau/nv04_graph.c
@@ -450,13 +450,13 @@ nv04_graph_context_del(struct nouveau_channel *chan, int engine)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-        nv04_graph_fifo_access(dev, false);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
 
         /* Unload the context if it's the currently active one */
         if (nv04_graph_channel(dev) == chan)
                 nv04_graph_unload_context(dev);
 
-        nv04_graph_fifo_access(dev, true);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
         /* Free the context resources */
@@ -538,24 +538,18 @@ nv04_graph_init(struct drm_device *dev, int engine)
 }
 
 static int
-nv04_graph_fini(struct drm_device *dev, int engine)
+nv04_graph_fini(struct drm_device *dev, int engine, bool suspend)
 {
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
+        if (!nv_wait(dev, NV04_PGRAPH_STATUS, ~0, 0) && suspend) {
+                nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+                return -EBUSY;
+        }
         nv04_graph_unload_context(dev);
         nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
         return 0;
 }
 
-void
-nv04_graph_fifo_access(struct drm_device *dev, bool enabled)
-{
-        if (enabled)
-                nv_wr32(dev, NV04_PGRAPH_FIFO,
-                                        nv_rd32(dev, NV04_PGRAPH_FIFO) | 1);
-        else
-                nv_wr32(dev, NV04_PGRAPH_FIFO,
-                                        nv_rd32(dev, NV04_PGRAPH_FIFO) & ~1);
-}
-
 static int
 nv04_graph_mthd_set_ref(struct nouveau_channel *chan,
                         u32 class, u32 mthd, u32 data)
diff --git a/drivers/gpu/drm/nouveau/nv04_instmem.c b/drivers/gpu/drm/nouveau/nv04_instmem.c
index b8611b95531..c1248e0740a 100644
--- a/drivers/gpu/drm/nouveau/nv04_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv04_instmem.c
@@ -28,6 +28,31 @@ int nv04_instmem_init(struct drm_device *dev)
         /* RAMIN always available */
         dev_priv->ramin_available = true;
 
+        /* Reserve space at end of VRAM for PRAMIN */
+        if (dev_priv->card_type >= NV_40) {
+                u32 vs = hweight8((nv_rd32(dev, 0x001540) & 0x0000ff00) >> 8);
+                u32 rsvd;
+
+                /* estimate grctx size, the magics come from nv40_grctx.c */
+                if      (dev_priv->chipset == 0x40) rsvd = 0x6aa0 * vs;
+                else if (dev_priv->chipset  < 0x43) rsvd = 0x4f00 * vs;
+                else if (nv44_graph_class(dev))     rsvd = 0x4980 * vs;
+                else                                rsvd = 0x4a40 * vs;
+                rsvd += 16 * 1024;
+                rsvd *= dev_priv->engine.fifo.channels;
+
+                /* pciegart table */
+                if (pci_is_pcie(dev->pdev))
+                        rsvd += 512 * 1024;
+
+                /* object storage */
+                rsvd += 512 * 1024;
+
+                dev_priv->ramin_rsvd_vram = round_up(rsvd, 4096);
+        } else {
+                dev_priv->ramin_rsvd_vram = 512 * 1024;
+        }
+
         /* Setup shared RAMHT */
         ret = nouveau_gpuobj_new_fake(dev, 0x10000, ~0, 4096,
                                       NVOBJ_FLAG_ZERO_ALLOC, &ramht);
@@ -112,7 +137,8 @@ nv04_instmem_resume(struct drm_device *dev)
 }
 
 int
-nv04_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
+nv04_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
+                 u32 size, u32 align)
 {
         struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
         struct drm_mm_node *ramin = NULL;
diff --git a/drivers/gpu/drm/nouveau/nv10_graph.c b/drivers/gpu/drm/nouveau/nv10_graph.c
index 0930c6cb88e..7255e4a4d3f 100644
--- a/drivers/gpu/drm/nouveau/nv10_graph.c
+++ b/drivers/gpu/drm/nouveau/nv10_graph.c
@@ -708,8 +708,8 @@ static void nv10_graph_load_dma_vtxbuf(struct nouveau_channel *chan,
                 0x2c000000 | chan->id << 20 | subchan << 16 | 0x18c);
         nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2_DL, inst);
         nv_mask(dev, NV10_PGRAPH_CTX_CONTROL, 0, 0x10000);
-        nv04_graph_fifo_access(dev, true);
-        nv04_graph_fifo_access(dev, false);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
 
         /* Restore the FIFO state */
         for (i = 0; i < ARRAY_SIZE(fifo); i++)
@@ -879,13 +879,13 @@ nv10_graph_context_del(struct nouveau_channel *chan, int engine)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-        nv04_graph_fifo_access(dev, false);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
 
         /* Unload the context if it's the currently active one */
         if (nv10_graph_channel(dev) == chan)
                 nv10_graph_unload_context(dev);
 
-        nv04_graph_fifo_access(dev, true);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
         /* Free the context resources */
@@ -957,8 +957,13 @@ nv10_graph_init(struct drm_device *dev, int engine)
 }
 
 static int
-nv10_graph_fini(struct drm_device *dev, int engine)
+nv10_graph_fini(struct drm_device *dev, int engine, bool suspend)
 {
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
+        if (!nv_wait(dev, NV04_PGRAPH_STATUS, ~0, 0) && suspend) {
+                nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+                return -EBUSY;
+        }
         nv10_graph_unload_context(dev);
         nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nv20_graph.c b/drivers/gpu/drm/nouveau/nv20_graph.c
index affc7d7dd02..183e37512ef 100644
--- a/drivers/gpu/drm/nouveau/nv20_graph.c
+++ b/drivers/gpu/drm/nouveau/nv20_graph.c
@@ -454,13 +454,13 @@ nv20_graph_context_del(struct nouveau_channel *chan, int engine)
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-        nv04_graph_fifo_access(dev, false);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
 
         /* Unload the context if it's the currently active one */
         if (nv10_graph_channel(dev) == chan)
                 nv20_graph_unload_context(dev);
 
-        nv04_graph_fifo_access(dev, true);
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
         /* Free the context resources */
@@ -654,8 +654,13 @@ nv30_graph_init(struct drm_device *dev, int engine)
 }
 
 int
-nv20_graph_fini(struct drm_device *dev, int engine)
+nv20_graph_fini(struct drm_device *dev, int engine, bool suspend)
 {
+        nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
+        if (!nv_wait(dev, NV04_PGRAPH_STATUS, ~0, 0) && suspend) {
+                nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+                return -EBUSY;
+        }
         nv20_graph_unload_context(dev);
         nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
         return 0;
@@ -753,6 +758,7 @@ nv20_graph_create(struct drm_device *dev)
                         break;
                 default:
                         NV_ERROR(dev, "PGRAPH: unknown chipset\n");
+                        kfree(pgraph);
                         return 0;
                 }
         } else {
@@ -774,6 +780,7 @@ nv20_graph_create(struct drm_device *dev)
                         break;
                 default:
                         NV_ERROR(dev, "PGRAPH: unknown chipset\n");
+                        kfree(pgraph);
                         return 0;
                 }
         }
diff --git a/drivers/gpu/drm/nouveau/nv40_graph.c b/drivers/gpu/drm/nouveau/nv40_graph.c
index 5beb01b8ace..ba14a93d8af 100644
--- a/drivers/gpu/drm/nouveau/nv40_graph.c
+++ b/drivers/gpu/drm/nouveau/nv40_graph.c
@@ -35,89 +35,6 @@ struct nv40_graph_engine {
         u32 grctx_size;
 };
 
-static struct nouveau_channel *
-nv40_graph_channel(struct drm_device *dev)
-{
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_gpuobj *grctx;
-        uint32_t inst;
-        int i;
-
-        inst = nv_rd32(dev, NV40_PGRAPH_CTXCTL_CUR);
-        if (!(inst & NV40_PGRAPH_CTXCTL_CUR_LOADED))
-                return NULL;
-        inst = (inst & NV40_PGRAPH_CTXCTL_CUR_INSTANCE) << 4;
-
-        for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
-                if (!dev_priv->channels.ptr[i])
-                        continue;
-
-                grctx = dev_priv->channels.ptr[i]->engctx[NVOBJ_ENGINE_GR];
-                if (grctx && grctx->pinst == inst)
-                        return dev_priv->channels.ptr[i];
-        }
-
-        return NULL;
-}
-
-static int
-nv40_graph_transfer_context(struct drm_device *dev, uint32_t inst, int save)
-{
-        uint32_t old_cp, tv = 1000, tmp;
-        int i;
-
-        old_cp = nv_rd32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER);
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, inst);
-
-        tmp  = nv_rd32(dev, NV40_PGRAPH_CTXCTL_0310);
-        tmp |= save ? NV40_PGRAPH_CTXCTL_0310_XFER_SAVE :
-                      NV40_PGRAPH_CTXCTL_0310_XFER_LOAD;
-        nv_wr32(dev, NV40_PGRAPH_CTXCTL_0310, tmp);
-
-        tmp  = nv_rd32(dev, NV40_PGRAPH_CTXCTL_0304);
-        tmp |= NV40_PGRAPH_CTXCTL_0304_XFER_CTX;
-        nv_wr32(dev, NV40_PGRAPH_CTXCTL_0304, tmp);
-
-        nouveau_wait_for_idle(dev);
-
-        for (i = 0; i < tv; i++) {
-                if (nv_rd32(dev, NV40_PGRAPH_CTXCTL_030C) == 0)
-                        break;
-        }
-
-        nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, old_cp);
-
-        if (i == tv) {
-                uint32_t ucstat = nv_rd32(dev, NV40_PGRAPH_CTXCTL_UCODE_STAT);
-                NV_ERROR(dev, "Failed: Instance=0x%08x Save=%d\n", inst, save);
-                NV_ERROR(dev, "IP: 0x%02x, Opcode: 0x%08x\n",
-                         ucstat >> NV40_PGRAPH_CTXCTL_UCODE_STAT_IP_SHIFT,
-                         ucstat  & NV40_PGRAPH_CTXCTL_UCODE_STAT_OP_MASK);
-                NV_ERROR(dev, "0x40030C = 0x%08x\n",
-                         nv_rd32(dev, NV40_PGRAPH_CTXCTL_030C));
-                return -EBUSY;
-        }
-
-        return 0;
-}
-
-static int
-nv40_graph_unload_context(struct drm_device *dev)
-{
-        uint32_t inst;
-        int ret;
-
-        inst = nv_rd32(dev, NV40_PGRAPH_CTXCTL_CUR);
-        if (!(inst & NV40_PGRAPH_CTXCTL_CUR_LOADED))
-                return 0;
-        inst &= NV40_PGRAPH_CTXCTL_CUR_INSTANCE;
-
-        ret = nv40_graph_transfer_context(dev, inst, 1);
-
-        nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, inst);
-        return ret;
-}
-
 static int
 nv40_graph_context_new(struct nouveau_channel *chan, int engine)
 {
@@ -163,16 +80,16 @@ nv40_graph_context_del(struct nouveau_channel *chan, int engine)
         struct nouveau_gpuobj *grctx = chan->engctx[engine];
         struct drm_device *dev = chan->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        u32 inst = 0x01000000 | (grctx->pinst >> 4);
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-        nv04_graph_fifo_access(dev, false);
-
-        /* Unload the context if it's the currently active one */
-        if (nv40_graph_channel(dev) == chan)
-                nv40_graph_unload_context(dev);
-
-        nv04_graph_fifo_access(dev, true);
+        nv_mask(dev, 0x400720, 0x00000000, 0x00000001);
+        if (nv_rd32(dev, 0x40032c) == inst)
+                nv_mask(dev, 0x40032c, 0x01000000, 0x00000000);
+        if (nv_rd32(dev, 0x400330) == inst)
+                nv_mask(dev, 0x400330, 0x01000000, 0x00000000);
+        nv_mask(dev, 0x400720, 0x00000001, 0x00000001);
         spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 
         /* Free the context resources */
@@ -429,9 +346,20 @@ nv40_graph_init(struct drm_device *dev, int engine)
 }
 
 static int
-nv40_graph_fini(struct drm_device *dev, int engine)
+nv40_graph_fini(struct drm_device *dev, int engine, bool suspend)
 {
-        nv40_graph_unload_context(dev);
+        u32 inst = nv_rd32(dev, 0x40032c);
+        if (inst & 0x01000000) {
+                nv_wr32(dev, 0x400720, 0x00000000);
+                nv_wr32(dev, 0x400784, inst);
+                nv_mask(dev, 0x400310, 0x00000020, 0x00000020);
+                nv_mask(dev, 0x400304, 0x00000001, 0x00000001);
+                if (!nv_wait(dev, 0x400300, 0x00000001, 0x00000000)) {
+                        u32 insn = nv_rd32(dev, 0x400308);
+                        NV_ERROR(dev, "PGRAPH: ctxprog timeout 0x%08x\n", insn);
+                }
+                nv_mask(dev, 0x40032c, 0x01000000, 0x00000000);
+        }
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nv40_mpeg.c b/drivers/gpu/drm/nouveau/nv40_mpeg.c
index 6d2af292a2e..ad03a0e1fc7 100644
--- a/drivers/gpu/drm/nouveau/nv40_mpeg.c
+++ b/drivers/gpu/drm/nouveau/nv40_mpeg.c
@@ -137,7 +137,7 @@ nv40_mpeg_init(struct drm_device *dev, int engine)
 }
 
 static int
-nv40_mpeg_fini(struct drm_device *dev, int engine)
+nv40_mpeg_fini(struct drm_device *dev, int engine, bool suspend)
 {
         /*XXX: context save? */
         nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
diff --git a/drivers/gpu/drm/nouveau/nv50_crtc.c b/drivers/gpu/drm/nouveau/nv50_crtc.c
index ebabacf38da..5d989073ba6 100644
--- a/drivers/gpu/drm/nouveau/nv50_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv50_crtc.c
@@ -104,7 +104,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
                 OUT_RING(evo, nv_crtc->lut.depth == 8 ?
                                 NV50_EVO_CRTC_CLUT_MODE_OFF :
                                 NV50_EVO_CRTC_CLUT_MODE_ON);
-                OUT_RING(evo, (nv_crtc->lut.nvbo->bo.mem.start << PAGE_SHIFT) >> 8);
+                OUT_RING(evo, nv_crtc->lut.nvbo->bo.offset >> 8);
                 if (dev_priv->chipset != 0x50) {
                         BEGIN_RING(evo, 0, NV84_EVO_CRTC(index, CLUT_DMA), 1);
                         OUT_RING(evo, NvEvoVRAM);
@@ -372,7 +372,7 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
 
         nouveau_bo_unmap(cursor);
 
-        nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT);
+        nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset);
         nv_crtc->cursor.show(nv_crtc, true);
 
 out:
@@ -519,12 +519,18 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
         struct drm_device *dev = nv_crtc->base.dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *evo = nv50_display(dev)->master;
-        struct drm_framebuffer *drm_fb = nv_crtc->base.fb;
-        struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb);
+        struct drm_framebuffer *drm_fb;
+        struct nouveau_framebuffer *fb;
         int ret;
 
         NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
 
+        /* no fb bound */
+        if (!atomic && !crtc->fb) {
+                NV_DEBUG_KMS(dev, "No FB bound\n");
+                return 0;
+        }
+
         /* If atomic, we want to switch to the fb we were passed, so
          * now we update pointers to do that.  (We don't pin; just
          * assume we're already pinned and update the base address.)
@@ -533,6 +539,8 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
                 drm_fb = passed_fb;
                 fb = nouveau_framebuffer(passed_fb);
         } else {
+                drm_fb = crtc->fb;
+                fb = nouveau_framebuffer(crtc->fb);
                 /* If not atomic, we can go ahead and pin, and unpin the
                  * old fb we were passed.
                  */
@@ -546,7 +554,7 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
                 }
         }
 
-        nv_crtc->fb.offset = fb->nvbo->bo.mem.start << PAGE_SHIFT;
+        nv_crtc->fb.offset = fb->nvbo->bo.offset;
         nv_crtc->fb.tile_flags = nouveau_bo_tile_layout(fb->nvbo);
         nv_crtc->fb.cpp = drm_fb->bits_per_pixel / 8;
         if (!nv_crtc->fb.blanked && dev_priv->chipset != 0x50) {
@@ -747,7 +755,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
         }
         nv_crtc->lut.depth = 0;
 
-        ret = nouveau_bo_new(dev, NULL, 4096, 0x100, TTM_PL_FLAG_VRAM,
+        ret = nouveau_bo_new(dev, 4096, 0x100, TTM_PL_FLAG_VRAM,
                              0, 0x0000, &nv_crtc->lut.nvbo);
         if (!ret) {
                 ret = nouveau_bo_pin(nv_crtc->lut.nvbo, TTM_PL_FLAG_VRAM);
@@ -773,7 +781,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
         drm_crtc_helper_add(&nv_crtc->base, &nv50_crtc_helper_funcs);
         drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
 
-        ret = nouveau_bo_new(dev, NULL, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
+        ret = nouveau_bo_new(dev, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
                              0, 0x0000, &nv_crtc->cursor.nvbo);
         if (!ret) {
                 ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 08da478ba54..db1a5f4b711 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -415,8 +415,6 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 
         /* synchronise with the rendering channel, if necessary */
         if (likely(chan)) {
-                u64 offset = dispc->sem.bo->vma.offset + dispc->sem.offset;
-
                 ret = RING_SPACE(chan, 10);
                 if (ret) {
                         WIND_RING(evo);
@@ -438,6 +436,8 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                         else
                                 OUT_RING  (chan, chan->vram_handle);
                 } else {
+                        u64 offset = chan->dispc_vma[nv_crtc->index].offset;
+                        offset += dispc->sem.offset;
                         BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
                         OUT_RING  (chan, upper_32_bits(offset));
                         OUT_RING  (chan, lower_32_bits(offset));
@@ -484,7 +484,7 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         OUT_RING  (evo, 0x00000000);
         OUT_RING  (evo, 0x00000000);
         BEGIN_RING(evo, 0, 0x0800, 5);
-        OUT_RING  (evo, (nv_fb->nvbo->bo.mem.start << PAGE_SHIFT) >> 8);
+        OUT_RING  (evo, nv_fb->nvbo->bo.offset >> 8);
         OUT_RING  (evo, 0);
         OUT_RING  (evo, (fb->height << 16) | fb->width);
         OUT_RING  (evo, nv_fb->r_pitch);
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.c b/drivers/gpu/drm/nouveau/nv50_evo.c
index c8e83c1a4de..c99d9751880 100644
--- a/drivers/gpu/drm/nouveau/nv50_evo.c
+++ b/drivers/gpu/drm/nouveau/nv50_evo.c
@@ -38,6 +38,7 @@ nv50_evo_channel_del(struct nouveau_channel **pevo)
                 return;
         *pevo = NULL;
 
+        nouveau_ramht_ref(NULL, &evo->ramht, evo);
         nouveau_gpuobj_channel_takedown(evo);
         nouveau_bo_unmap(evo->pushbuf_bo);
         nouveau_bo_ref(NULL, &evo->pushbuf_bo);
@@ -116,7 +117,7 @@ nv50_evo_channel_new(struct drm_device *dev, int chid,
         evo->user_get = 4;
         evo->user_put = 0;
 
-        ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
+        ret = nouveau_bo_new(dev, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
                              &evo->pushbuf_bo);
         if (ret == 0)
                 ret = nouveau_bo_pin(evo->pushbuf_bo, TTM_PL_FLAG_VRAM);
@@ -153,7 +154,7 @@ nv50_evo_channel_init(struct nouveau_channel *evo)
 {
         struct drm_device *dev = evo->dev;
         int id = evo->id, ret, i;
-        u64 pushbuf = evo->pushbuf_bo->bo.mem.start << PAGE_SHIFT;
+        u64 pushbuf = evo->pushbuf_bo->bo.offset;
         u32 tmp;
 
         tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
@@ -331,16 +332,15 @@ nv50_evo_create(struct drm_device *dev)
                 if (ret)
                         goto err;
 
-                ret = nouveau_bo_new(dev, NULL, 4096, 0x1000, TTM_PL_FLAG_VRAM,
+                ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
                                      0, 0x0000, &dispc->sem.bo);
                 if (!ret) {
-                        offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
-
                         ret = nouveau_bo_pin(dispc->sem.bo, TTM_PL_FLAG_VRAM);
                         if (!ret)
                                 ret = nouveau_bo_map(dispc->sem.bo);
                         if (ret)
                                 nouveau_bo_ref(NULL, &dispc->sem.bo);
+                        offset = dispc->sem.bo->bo.offset;
                 }
 
                 if (ret)
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index 791ded1c5c6..dc75a720652 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -159,7 +159,7 @@ nv50_fbcon_accel_init(struct fb_info *info)
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
-        struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
+        struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
         int ret, format;
 
         switch (info->var.bits_per_pixel) {
@@ -247,8 +247,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
         OUT_RING(chan, info->fix.line_length);
         OUT_RING(chan, info->var.xres_virtual);
         OUT_RING(chan, info->var.yres_virtual);
-        OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
-        OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, upper_32_bits(fb->vma.offset));
+        OUT_RING(chan, lower_32_bits(fb->vma.offset));
         BEGIN_RING(chan, NvSub2D, 0x0230, 2);
         OUT_RING(chan, format);
         OUT_RING(chan, 1);
@@ -256,8 +256,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
         OUT_RING(chan, info->fix.line_length);
         OUT_RING(chan, info->var.xres_virtual);
         OUT_RING(chan, info->var.yres_virtual);
-        OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
-        OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, upper_32_bits(fb->vma.offset));
+        OUT_RING(chan, lower_32_bits(fb->vma.offset));
 
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_graph.c b/drivers/gpu/drm/nouveau/nv50_graph.c
index e25cbb46789..d43c46caa76 100644
--- a/drivers/gpu/drm/nouveau/nv50_graph.c
+++ b/drivers/gpu/drm/nouveau/nv50_graph.c
@@ -31,7 +31,6 @@
 #include "nouveau_grctx.h"
 #include "nouveau_dma.h"
 #include "nouveau_vm.h"
-#include "nouveau_ramht.h"
 #include "nv50_evo.h"
 
 struct nv50_graph_engine {
@@ -125,7 +124,6 @@ static void
 nv50_graph_init_reset(struct drm_device *dev)
 {
         uint32_t pmc_e = NV_PMC_ENABLE_PGRAPH | (1 << 21);
-
         NV_DEBUG(dev, "\n");
 
         nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & ~pmc_e);
@@ -255,9 +253,13 @@ nv50_graph_init(struct drm_device *dev, int engine)
 }
 
 static int
-nv50_graph_fini(struct drm_device *dev, int engine)
+nv50_graph_fini(struct drm_device *dev, int engine, bool suspend)
 {
-        NV_DEBUG(dev, "\n");
+        nv_mask(dev, 0x400500, 0x00010001, 0x00000000);
+        if (!nv_wait(dev, 0x400700, ~0, 0) && suspend) {
+                nv_mask(dev, 0x400500, 0x00010001, 0x00010001);
+                return -EBUSY;
+        }
         nv50_graph_unload_context(dev);
         nv_wr32(dev, 0x40013c, 0x00000000);
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c
index 4f95a1e5822..a7c12c94a5a 100644
--- a/drivers/gpu/drm/nouveau/nv50_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv50_instmem.c
@@ -305,9 +305,9 @@ struct nv50_gpuobj_node {
         u32 align;
 };
 
-
 int
-nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
+nv50_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
+                 u32 size, u32 align)
 {
         struct drm_device *dev = gpuobj->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -336,7 +336,7 @@ nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
                 if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER))
                         flags |= NV_MEM_ACCESS_SYS;
 
-                ret = nouveau_vm_get(dev_priv->chan_vm, size, 12, flags,
+                ret = nouveau_vm_get(chan->vm, size, 12, flags,
                                      &node->chan_vma);
                 if (ret) {
                         vram->put(dev, &node->vram);
@@ -345,7 +345,7 @@ nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
                 }
 
                 nouveau_vm_map(&node->chan_vma, node->vram);
-                gpuobj->vinst = node->chan_vma.offset;
+                gpuobj->linst = node->chan_vma.offset;
         }
 
         gpuobj->size = size;
diff --git a/drivers/gpu/drm/nouveau/nv50_mpeg.c b/drivers/gpu/drm/nouveau/nv50_mpeg.c
index 1dc5913f78c..b57a2d180ad 100644
--- a/drivers/gpu/drm/nouveau/nv50_mpeg.c
+++ b/drivers/gpu/drm/nouveau/nv50_mpeg.c
@@ -160,7 +160,7 @@ nv50_mpeg_init(struct drm_device *dev, int engine)
 }
 
 static int
-nv50_mpeg_fini(struct drm_device *dev, int engine)
+nv50_mpeg_fini(struct drm_device *dev, int engine, bool suspend)
 {
         /*XXX: context save for s/r */
         nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
diff --git a/drivers/gpu/drm/nouveau/nv50_sor.c b/drivers/gpu/drm/nouveau/nv50_sor.c
index c25c5938642..ffe8b483b7b 100644
--- a/drivers/gpu/drm/nouveau/nv50_sor.c
+++ b/drivers/gpu/drm/nouveau/nv50_sor.c
@@ -318,6 +318,8 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_entry *entry)
                 uint32_t tmp;
 
                 tmp = nv_rd32(dev, 0x61c700 + (or * 0x800));
+                if (!tmp)
+                        tmp = nv_rd32(dev, 0x610798 + (or * 8));
 
                 switch ((tmp & 0x00000f00) >> 8) {
                 case 8:
diff --git a/drivers/gpu/drm/nouveau/nv50_vm.c b/drivers/gpu/drm/nouveau/nv50_vm.c
index 1a0dd491a0e..40b84f22d81 100644
--- a/drivers/gpu/drm/nouveau/nv50_vm.c
+++ b/drivers/gpu/drm/nouveau/nv50_vm.c
@@ -156,7 +156,7 @@ nv50_vm_flush(struct nouveau_vm *vm)
         pinstmem->flush(vm->dev);
 
         /* BAR */
-        if (vm != dev_priv->chan_vm) {
+        if (vm == dev_priv->bar1_vm || vm == dev_priv->bar3_vm) {
                 nv50_vm_flush_engine(vm->dev, 6);
                 return;
         }
diff --git a/drivers/gpu/drm/nouveau/nv50_vram.c b/drivers/gpu/drm/nouveau/nv50_vram.c
index ffbc3d8cf5b..af32daecd1e 100644
--- a/drivers/gpu/drm/nouveau/nv50_vram.c
+++ b/drivers/gpu/drm/nouveau/nv50_vram.c
@@ -51,9 +51,7 @@ void
 nv50_vram_del(struct drm_device *dev, struct nouveau_mem **pmem)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
-        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
-        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm *mm = dev_priv->engine.vram.mm;
         struct nouveau_mm_node *this;
         struct nouveau_mem *mem;
 
@@ -84,9 +82,7 @@ nv50_vram_new(struct drm_device *dev, u64 size, u32 align, u32 size_nc,
               u32 memtype, struct nouveau_mem **pmem)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
-        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
-        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm *mm = dev_priv->engine.vram.mm;
         struct nouveau_mm_node *r;
         struct nouveau_mem *mem;
         int comp = (memtype & 0x300) >> 8;
@@ -190,22 +186,35 @@ int
 nv50_vram_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+        const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+        u32 rblock, length;
 
         dev_priv->vram_size  = nv_rd32(dev, 0x10020c);
         dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
         dev_priv->vram_size &= 0xffffffff00ULL;
 
-        switch (dev_priv->chipset) {
-        case 0xaa:
-        case 0xac:
-        case 0xaf:
+        /* IGPs, no funky reordering happens here, they don't have VRAM */
+        if (dev_priv->chipset == 0xaa ||
+            dev_priv->chipset == 0xac ||
+            dev_priv->chipset == 0xaf) {
                 dev_priv->vram_sys_base = (u64)nv_rd32(dev, 0x100e10) << 12;
-                dev_priv->vram_rblock_size = 4096;
-                break;
-        default:
-                dev_priv->vram_rblock_size = nv50_vram_rblock(dev);
-                break;
+                rblock = 4096 >> 12;
+        } else {
+                rblock = nv50_vram_rblock(dev) >> 12;
         }
 
-        return 0;
+        length = (dev_priv->vram_size >> 12) - rsvd_head - rsvd_tail;
+
+        return nouveau_mm_init(&vram->mm, rsvd_head, length, rblock);
+}
+
+void
+nv50_vram_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+
+        nouveau_mm_fini(&vram->mm);
 }
diff --git a/drivers/gpu/drm/nouveau/nv84_crypt.c b/drivers/gpu/drm/nouveau/nv84_crypt.c
index 75b809a5174..edece9c616e 100644
--- a/drivers/gpu/drm/nouveau/nv84_crypt.c
+++ b/drivers/gpu/drm/nouveau/nv84_crypt.c
@@ -138,7 +138,7 @@ nv84_crypt_isr(struct drm_device *dev)
 }
 
 static int
-nv84_crypt_fini(struct drm_device *dev, int engine)
+nv84_crypt_fini(struct drm_device *dev, int engine, bool suspend)
 {
         nv_wr32(dev, 0x102140, 0x00000000);
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.c b/drivers/gpu/drm/nouveau/nva3_copy.c
index b86820a6122..8f356d58e40 100644
--- a/drivers/gpu/drm/nouveau/nva3_copy.c
+++ b/drivers/gpu/drm/nouveau/nva3_copy.c
@@ -140,7 +140,7 @@ nva3_copy_init(struct drm_device *dev, int engine)
 }
 
 static int
-nva3_copy_fini(struct drm_device *dev, int engine)
+nva3_copy_fini(struct drm_device *dev, int engine, bool suspend)
 {
         nv_mask(dev, 0x104048, 0x00000003, 0x00000000);
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_copy.c b/drivers/gpu/drm/nouveau/nvc0_copy.c
index 208fa7ab3f4..dddf006f6d8 100644
--- a/drivers/gpu/drm/nouveau/nvc0_copy.c
+++ b/drivers/gpu/drm/nouveau/nvc0_copy.c
@@ -48,14 +48,14 @@ nvc0_copy_context_new(struct nouveau_channel *chan, int engine)
         struct nouveau_gpuobj *ctx = NULL;
         int ret;
 
-        ret = nouveau_gpuobj_new(dev, NULL, 256, 256,
+        ret = nouveau_gpuobj_new(dev, chan, 256, 256,
                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER |
                                  NVOBJ_FLAG_ZERO_ALLOC, &ctx);
         if (ret)
                 return ret;
 
-        nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(ctx->vinst));
-        nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(ctx->vinst));
+        nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(ctx->linst));
+        nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(ctx->linst));
         dev_priv->engine.instmem.flush(dev);
 
         chan->engctx[engine] = ctx;
@@ -127,7 +127,7 @@ nvc0_copy_init(struct drm_device *dev, int engine)
 }
 
 static int
-nvc0_copy_fini(struct drm_device *dev, int engine)
+nvc0_copy_fini(struct drm_device *dev, int engine, bool suspend)
 {
         struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_fb.c b/drivers/gpu/drm/nouveau/nvc0_fb.c
index 26a996025dd..08e6b118f02 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fb.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2010 Red Hat Inc.
+ * Copyright 2011 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -23,16 +23,80 @@
  */
 
 #include "drmP.h"
-
+#include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_drm.h"
+
+struct nvc0_fb_priv {
+        struct page *r100c10_page;
+        dma_addr_t r100c10;
+};
+
+static void
+nvc0_fb_destroy(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nvc0_fb_priv *priv = pfb->priv;
+
+        if (priv->r100c10_page) {
+                pci_unmap_page(dev->pdev, priv->r100c10, PAGE_SIZE,
+                               PCI_DMA_BIDIRECTIONAL);
+                __free_page(priv->r100c10_page);
+        }
+
+        kfree(priv);
+        pfb->priv = NULL;
+}
+
+static int
+nvc0_fb_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nvc0_fb_priv *priv;
+
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        pfb->priv = priv;
+
+        priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+        if (!priv->r100c10_page) {
+                nvc0_fb_destroy(dev);
+                return -ENOMEM;
+        }
+
+        priv->r100c10 = pci_map_page(dev->pdev, priv->r100c10_page, 0,
+                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        if (pci_dma_mapping_error(dev->pdev, priv->r100c10)) {
+                nvc0_fb_destroy(dev);
+                return -EFAULT;
+        }
+
+        return 0;
+}
 
 int
 nvc0_fb_init(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvc0_fb_priv *priv;
+        int ret;
+
+        if (!dev_priv->engine.fb.priv) {
+                ret = nvc0_fb_create(dev);
+                if (ret)
+                        return ret;
+        }
+        priv = dev_priv->engine.fb.priv;
+
+        nv_wr32(dev, 0x100c10, priv->r100c10 >> 8);
         return 0;
 }
 
 void
 nvc0_fb_takedown(struct drm_device *dev)
 {
+        nvc0_fb_destroy(dev);
 }
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
index fa5d4c23438..a495e48197c 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -159,7 +159,7 @@ nvc0_fbcon_accel_init(struct fb_info *info)
         struct drm_device *dev = nfbdev->dev;
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nouveau_channel *chan = dev_priv->channel;
-        struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
+        struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
         int ret, format;
 
         ret = nouveau_gpuobj_gr_new(chan, 0x902d, 0x902d);
@@ -203,8 +203,8 @@ nvc0_fbcon_accel_init(struct fb_info *info)
         BEGIN_NVC0(chan, 2, NvSub2D, 0x0000, 1);
         OUT_RING  (chan, 0x0000902d);
         BEGIN_NVC0(chan, 2, NvSub2D, 0x0104, 2);
-        OUT_RING  (chan, upper_32_bits(chan->notifier_bo->bo.offset));
-        OUT_RING  (chan, lower_32_bits(chan->notifier_bo->bo.offset));
+        OUT_RING  (chan, upper_32_bits(chan->notifier_vma.offset));
+        OUT_RING  (chan, lower_32_bits(chan->notifier_vma.offset));
         BEGIN_NVC0(chan, 2, NvSub2D, 0x0290, 1);
         OUT_RING  (chan, 0);
         BEGIN_NVC0(chan, 2, NvSub2D, 0x0888, 1);
@@ -249,8 +249,8 @@ nvc0_fbcon_accel_init(struct fb_info *info)
         OUT_RING  (chan, info->fix.line_length);
         OUT_RING  (chan, info->var.xres_virtual);
         OUT_RING  (chan, info->var.yres_virtual);
-        OUT_RING  (chan, upper_32_bits(nvbo->vma.offset));
-        OUT_RING  (chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, upper_32_bits(fb->vma.offset));
+        OUT_RING  (chan, lower_32_bits(fb->vma.offset));
         BEGIN_NVC0(chan, 2, NvSub2D, 0x0230, 10);
         OUT_RING  (chan, format);
         OUT_RING  (chan, 1);
@@ -260,8 +260,8 @@ nvc0_fbcon_accel_init(struct fb_info *info)
         OUT_RING  (chan, info->fix.line_length);
         OUT_RING  (chan, info->var.xres_virtual);
         OUT_RING  (chan, info->var.yres_virtual);
-        OUT_RING  (chan, upper_32_bits(nvbo->vma.offset));
-        OUT_RING  (chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, upper_32_bits(fb->vma.offset));
+        OUT_RING  (chan, lower_32_bits(fb->vma.offset));
         FIRE_RING (chan);
 
         return 0;
diff --git a/drivers/gpu/drm/nouveau/nvc0_fifo.c b/drivers/gpu/drm/nouveau/nvc0_fifo.c
index fb4f5943e01..6f9f341c3e8 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fifo.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fifo.c
@@ -210,10 +210,10 @@ nvc0_fifo_unload_context(struct drm_device *dev)
         int i;
 
         for (i = 0; i < 128; i++) {
-                if (!(nv_rd32(dev, 0x003004 + (i * 4)) & 1))
+                if (!(nv_rd32(dev, 0x003004 + (i * 8)) & 1))
                         continue;
 
-                nv_mask(dev, 0x003004 + (i * 4), 0x00000001, 0x00000000);
+                nv_mask(dev, 0x003004 + (i * 8), 0x00000001, 0x00000000);
                 nv_wr32(dev, 0x002634, i);
                 if (!nv_wait(dev, 0x002634, 0xffffffff, i)) {
                         NV_INFO(dev, "PFIFO: kick ch %d failed: 0x%08x\n",
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.c b/drivers/gpu/drm/nouveau/nvc0_graph.c
index ca6db204d64..5b2f6f42046 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.c
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.c
@@ -28,7 +28,34 @@
 
 #include "nouveau_drv.h"
 #include "nouveau_mm.h"
+
 #include "nvc0_graph.h"
+#include "nvc0_grhub.fuc.h"
+#include "nvc0_grgpc.fuc.h"
+
+static void
+nvc0_graph_ctxctl_debug_unit(struct drm_device *dev, u32 base)
+{
+        NV_INFO(dev, "PGRAPH: %06x - done 0x%08x\n", base,
+                nv_rd32(dev, base + 0x400));
+        NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
+                nv_rd32(dev, base + 0x800), nv_rd32(dev, base + 0x804),
+                nv_rd32(dev, base + 0x808), nv_rd32(dev, base + 0x80c));
+        NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
+                nv_rd32(dev, base + 0x810), nv_rd32(dev, base + 0x814),
+                nv_rd32(dev, base + 0x818), nv_rd32(dev, base + 0x81c));
+}
+
+static void
+nvc0_graph_ctxctl_debug(struct drm_device *dev)
+{
+        u32 gpcnr = nv_rd32(dev, 0x409604) & 0xffff;
+        u32 gpc;
+
+        nvc0_graph_ctxctl_debug_unit(dev, 0x409000);
+        for (gpc = 0; gpc < gpcnr; gpc++)
+                nvc0_graph_ctxctl_debug_unit(dev, 0x502000 + (gpc * 0x8000));
+}
 
 static int
 nvc0_graph_load_context(struct nouveau_channel *chan)
@@ -72,24 +99,44 @@ nvc0_graph_construct_context(struct nouveau_channel *chan)
         if (!ctx)
                 return -ENOMEM;
 
-        nvc0_graph_load_context(chan);
-
-        nv_wo32(grch->grctx, 0x1c, 1);
-        nv_wo32(grch->grctx, 0x20, 0);
-        nv_wo32(grch->grctx, 0x28, 0);
-        nv_wo32(grch->grctx, 0x2c, 0);
-        dev_priv->engine.instmem.flush(dev);
-
-        ret = nvc0_grctx_generate(chan);
-        if (ret) {
-                kfree(ctx);
-                return ret;
+        if (!nouveau_ctxfw) {
+                nv_wr32(dev, 0x409840, 0x80000000);
+                nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
+                nv_wr32(dev, 0x409504, 0x00000001);
+                if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
+                        NV_ERROR(dev, "PGRAPH: HUB_SET_CHAN timeout\n");
+                        nvc0_graph_ctxctl_debug(dev);
+                        ret = -EBUSY;
+                        goto err;
+                }
+        } else {
+                nvc0_graph_load_context(chan);
+
+                nv_wo32(grch->grctx, 0x1c, 1);
+                nv_wo32(grch->grctx, 0x20, 0);
+                nv_wo32(grch->grctx, 0x28, 0);
+                nv_wo32(grch->grctx, 0x2c, 0);
+                dev_priv->engine.instmem.flush(dev);
         }
 
-        ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
-        if (ret) {
-                kfree(ctx);
-                return ret;
+        ret = nvc0_grctx_generate(chan);
+        if (ret)
+                goto err;
+
+        if (!nouveau_ctxfw) {
+                nv_wr32(dev, 0x409840, 0x80000000);
+                nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
+                nv_wr32(dev, 0x409504, 0x00000002);
+                if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
+                        NV_ERROR(dev, "PGRAPH: HUB_CTX_SAVE timeout\n");
+                        nvc0_graph_ctxctl_debug(dev);
+                        ret = -EBUSY;
+                        goto err;
+                }
+        } else {
+                ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
+                if (ret)
+                        goto err;
         }
 
         for (i = 0; i < priv->grctx_size; i += 4)
@@ -97,6 +144,10 @@ nvc0_graph_construct_context(struct nouveau_channel *chan)
 
         priv->grctx_vals = ctx;
         return 0;
+
+err:
+        kfree(ctx);
+        return ret;
 }
 
 static int
@@ -108,50 +159,50 @@ nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
         int i = 0, gpc, tp, ret;
         u32 magic;
 
-        ret = nouveau_gpuobj_new(dev, NULL, 0x2000, 256, NVOBJ_FLAG_VM,
+        ret = nouveau_gpuobj_new(dev, chan, 0x2000, 256, NVOBJ_FLAG_VM,
                                  &grch->unk408004);
         if (ret)
                 return ret;
 
-        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 256, NVOBJ_FLAG_VM,
+        ret = nouveau_gpuobj_new(dev, chan, 0x8000, 256, NVOBJ_FLAG_VM,
                                  &grch->unk40800c);
         if (ret)
                 return ret;
 
-        ret = nouveau_gpuobj_new(dev, NULL, 384 * 1024, 4096,
+        ret = nouveau_gpuobj_new(dev, chan, 384 * 1024, 4096,
                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
                                  &grch->unk418810);
         if (ret)
                 return ret;
 
-        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0, NVOBJ_FLAG_VM,
+        ret = nouveau_gpuobj_new(dev, chan, 0x1000, 0, NVOBJ_FLAG_VM,
                                  &grch->mmio);
         if (ret)
                 return ret;
 
 
         nv_wo32(grch->mmio, i++ * 4, 0x00408004);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
         nv_wo32(grch->mmio, i++ * 4, 0x00408008);
         nv_wo32(grch->mmio, i++ * 4, 0x80000018);
 
         nv_wo32(grch->mmio, i++ * 4, 0x0040800c);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
         nv_wo32(grch->mmio, i++ * 4, 0x00408010);
         nv_wo32(grch->mmio, i++ * 4, 0x80000000);
 
         nv_wo32(grch->mmio, i++ * 4, 0x00418810);
-        nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->vinst >> 12);
+        nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->linst >> 12);
         nv_wo32(grch->mmio, i++ * 4, 0x00419848);
-        nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->vinst >> 12);
+        nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->linst >> 12);
 
         nv_wo32(grch->mmio, i++ * 4, 0x00419004);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
         nv_wo32(grch->mmio, i++ * 4, 0x00419008);
         nv_wo32(grch->mmio, i++ * 4, 0x00000000);
 
         nv_wo32(grch->mmio, i++ * 4, 0x00418808);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
         nv_wo32(grch->mmio, i++ * 4, 0x0041880c);
         nv_wo32(grch->mmio, i++ * 4, 0x80000018);
 
@@ -159,7 +210,7 @@ nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
         nv_wo32(grch->mmio, i++ * 4, 0x00405830);
         nv_wo32(grch->mmio, i++ * 4, magic);
         for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-                for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x02fc) {
+                for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x0324) {
                         u32 reg = 0x504520 + (gpc * 0x8000) + (tp * 0x0800);
                         nv_wo32(grch->mmio, i++ * 4, reg);
                         nv_wo32(grch->mmio, i++ * 4, magic);
@@ -186,7 +237,7 @@ nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
                 return -ENOMEM;
         chan->engctx[NVOBJ_ENGINE_GR] = grch;
 
-        ret = nouveau_gpuobj_new(dev, NULL, priv->grctx_size, 256,
+        ret = nouveau_gpuobj_new(dev, chan, priv->grctx_size, 256,
                                  NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
                                  &grch->grctx);
         if (ret)
@@ -197,8 +248,8 @@ nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
         if (ret)
                 goto error;
 
-        nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->vinst) | 4);
-        nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->vinst));
+        nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->linst) | 4);
+        nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->linst));
         pinstmem->flush(dev);
 
         if (!priv->grctx_vals) {
@@ -210,15 +261,20 @@ nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
         for (i = 0; i < priv->grctx_size; i += 4)
                 nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
 
-        nv_wo32(grctx, 0xf4, 0);
-        nv_wo32(grctx, 0xf8, 0);
-        nv_wo32(grctx, 0x10, grch->mmio_nr);
-        nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->vinst));
-        nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->vinst));
-        nv_wo32(grctx, 0x1c, 1);
-        nv_wo32(grctx, 0x20, 0);
-        nv_wo32(grctx, 0x28, 0);
-        nv_wo32(grctx, 0x2c, 0);
+        if (!nouveau_ctxfw) {
+                nv_wo32(grctx, 0x00, grch->mmio_nr);
+                nv_wo32(grctx, 0x04, grch->mmio->linst >> 8);
+        } else {
+                nv_wo32(grctx, 0xf4, 0);
+                nv_wo32(grctx, 0xf8, 0);
+                nv_wo32(grctx, 0x10, grch->mmio_nr);
+                nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->linst));
+                nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->linst));
+                nv_wo32(grctx, 0x1c, 1);
+                nv_wo32(grctx, 0x20, 0);
+                nv_wo32(grctx, 0x28, 0);
+                nv_wo32(grctx, 0x2c, 0);
+        }
         pinstmem->flush(dev);
         return 0;
 
@@ -248,7 +304,7 @@ nvc0_graph_object_new(struct nouveau_channel *chan, int engine,
 }
 
 static int
-nvc0_graph_fini(struct drm_device *dev, int engine)
+nvc0_graph_fini(struct drm_device *dev, int engine, bool suspend)
 {
         return 0;
 }
@@ -296,6 +352,7 @@ static void
 nvc0_graph_init_gpc_0(struct drm_device *dev)
 {
         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tp_total);
         u32 data[TP_MAX / 8];
         u8  tpnr[GPC_MAX];
         int i, gpc, tpc;
@@ -307,13 +364,6 @@ nvc0_graph_init_gpc_0(struct drm_device *dev)
          * 465: 3/4/4/0 4        7
          * 470: 3/3/4/4 5        5
          * 480: 3/4/4/4 6        6
-         *
-         * magicgpc918
-         * 450: 00200000 00000000001000000000000000000000
-         * 460: 00124925 00000000000100100100100100100101
-         * 465: 000ba2e9 00000000000010111010001011101001
-         * 470: 00092493 00000000000010010010010010010011
-         * 480: 00088889 00000000000010001000100010001001
          */
 
         memset(data, 0x00, sizeof(data));
@@ -336,10 +386,10 @@ nvc0_graph_init_gpc_0(struct drm_device *dev)
                 nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
                                                   priv->tp_nr[gpc]);
                 nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tp_total);
-                nv_wr32(dev, GPC_UNIT(gpc, 0x0918), priv->magicgpc918);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0918), magicgpc918);
         }
 
-        nv_wr32(dev, GPC_BCAST(0x1bd4), priv->magicgpc918);
+        nv_wr32(dev, GPC_BCAST(0x1bd4), magicgpc918);
         nv_wr32(dev, GPC_BCAST(0x08ac), priv->rop_nr);
 }
 
@@ -419,8 +469,51 @@ nvc0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
 static int
 nvc0_graph_init_ctxctl(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
         u32 r000260;
+        int i;
+
+        if (!nouveau_ctxfw) {
+                /* load HUB microcode */
+                r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
+                nv_wr32(dev, 0x4091c0, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
+                        nv_wr32(dev, 0x4091c4, nvc0_grhub_data[i]);
+
+                nv_wr32(dev, 0x409180, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
+                        if ((i & 0x3f) == 0)
+                                nv_wr32(dev, 0x409188, i >> 6);
+                        nv_wr32(dev, 0x409184, nvc0_grhub_code[i]);
+                }
+
+                /* load GPC microcode */
+                nv_wr32(dev, 0x41a1c0, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
+                        nv_wr32(dev, 0x41a1c4, nvc0_grgpc_data[i]);
+
+                nv_wr32(dev, 0x41a180, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
+                        if ((i & 0x3f) == 0)
+                                nv_wr32(dev, 0x41a188, i >> 6);
+                        nv_wr32(dev, 0x41a184, nvc0_grgpc_code[i]);
+                }
+                nv_wr32(dev, 0x000260, r000260);
+
+                /* start HUB ucode running, it'll init the GPCs */
+                nv_wr32(dev, 0x409800, dev_priv->chipset);
+                nv_wr32(dev, 0x40910c, 0x00000000);
+                nv_wr32(dev, 0x409100, 0x00000002);
+                if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
+                        NV_ERROR(dev, "PGRAPH: HUB_INIT timed out\n");
+                        nvc0_graph_ctxctl_debug(dev);
+                        return -EBUSY;
+                }
+
+                priv->grctx_size = nv_rd32(dev, 0x409804);
+                return 0;
+        }
 
         /* load fuc microcode */
         r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
@@ -528,6 +621,22 @@ nvc0_graph_isr_chid(struct drm_device *dev, u64 inst)
 }
 
 static void
+nvc0_graph_ctxctl_isr(struct drm_device *dev)
+{
+        u32 ustat = nv_rd32(dev, 0x409c18);
+
+        if (ustat & 0x00000001)
+                NV_INFO(dev, "PGRAPH: CTXCTRL ucode error\n");
+        if (ustat & 0x00080000)
+                NV_INFO(dev, "PGRAPH: CTXCTRL watchdog timeout\n");
+        if (ustat & ~0x00080001)
+                NV_INFO(dev, "PGRAPH: CTXCTRL 0x%08x\n", ustat);
+
+        nvc0_graph_ctxctl_debug(dev);
+        nv_wr32(dev, 0x409c20, ustat);
+}
+
+static void
 nvc0_graph_isr(struct drm_device *dev)
 {
         u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
@@ -578,11 +687,7 @@ nvc0_graph_isr(struct drm_device *dev)
         }
 
         if (stat & 0x00080000) {
-                u32 ustat = nv_rd32(dev, 0x409c18);
-
-                NV_INFO(dev, "PGRAPH: CTXCTRL ustat 0x%08x\n", ustat);
-
-                nv_wr32(dev, 0x409c20, ustat);
+                nvc0_graph_ctxctl_isr(dev);
                 nv_wr32(dev, 0x400100, 0x00080000);
                 stat &= ~0x00080000;
         }
@@ -606,7 +711,7 @@ nvc0_runk140_isr(struct drm_device *dev)
                 u32 st0 = nv_mask(dev, reg + 0x1020, 0, 0);
                 u32 st1 = nv_mask(dev, reg + 0x1420, 0, 0);
 
-                NV_INFO(dev, "PRUNK140: %d 0x%08x 0x%08x\n", unit, st0, st1);
+                NV_DEBUG(dev, "PRUNK140: %d 0x%08x 0x%08x\n", unit, st0, st1);
                 units &= ~(1 << unit);
         }
 }
@@ -651,10 +756,12 @@ nvc0_graph_destroy(struct drm_device *dev, int engine)
 {
         struct nvc0_graph_priv *priv = nv_engine(dev, engine);
 
-        nvc0_graph_destroy_fw(&priv->fuc409c);
-        nvc0_graph_destroy_fw(&priv->fuc409d);
-        nvc0_graph_destroy_fw(&priv->fuc41ac);
-        nvc0_graph_destroy_fw(&priv->fuc41ad);
+        if (nouveau_ctxfw) {
+                nvc0_graph_destroy_fw(&priv->fuc409c);
+                nvc0_graph_destroy_fw(&priv->fuc409d);
+                nvc0_graph_destroy_fw(&priv->fuc41ac);
+                nvc0_graph_destroy_fw(&priv->fuc41ad);
+        }
 
         nouveau_irq_unregister(dev, 12);
         nouveau_irq_unregister(dev, 25);
@@ -675,13 +782,10 @@ nvc0_graph_create(struct drm_device *dev)
         struct drm_nouveau_private *dev_priv = dev->dev_private;
         struct nvc0_graph_priv *priv;
         int ret, gpc, i;
+        u32 fermi;
 
-        switch (dev_priv->chipset) {
-        case 0xc0:
-        case 0xc3:
-        case 0xc4:
-                break;
-        default:
+        fermi = nvc0_graph_class(dev);
+        if (!fermi) {
                 NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
                 return 0;
         }
@@ -701,15 +805,17 @@ nvc0_graph_create(struct drm_device *dev)
         nouveau_irq_register(dev, 12, nvc0_graph_isr);
         nouveau_irq_register(dev, 25, nvc0_runk140_isr);
 
-        if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
-            nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
-            nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
-            nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
-                ret = 0;
-                goto error;
+        if (nouveau_ctxfw) {
+                NV_INFO(dev, "PGRAPH: using external firmware\n");
+                if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
+                    nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
+                    nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
+                    nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
+                        ret = 0;
+                        goto error;
+                }
         }
 
-
         ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
         if (ret)
                 goto error;
@@ -735,25 +841,28 @@ nvc0_graph_create(struct drm_device *dev)
         case 0xc0:
                 if (priv->tp_total == 11) { /* 465, 3/4/4/0, 4 */
                         priv->magic_not_rop_nr = 0x07;
-                        /* filled values up to tp_total, the rest 0 */
-                        priv->magicgpc918      = 0x000ba2e9;
                 } else
                 if (priv->tp_total == 14) { /* 470, 3/3/4/4, 5 */
                         priv->magic_not_rop_nr = 0x05;
-                        priv->magicgpc918      = 0x00092493;
                 } else
                 if (priv->tp_total == 15) { /* 480, 3/4/4/4, 6 */
                         priv->magic_not_rop_nr = 0x06;
-                        priv->magicgpc918      = 0x00088889;
                 }
                 break;
         case 0xc3: /* 450, 4/0/0/0, 2 */
                 priv->magic_not_rop_nr = 0x03;
-                priv->magicgpc918      = 0x00200000;
                 break;
         case 0xc4: /* 460, 3/4/0/0, 4 */
                 priv->magic_not_rop_nr = 0x01;
-                priv->magicgpc918      = 0x00124925;
+                break;
+        case 0xc1: /* 2/0/0/0, 1 */
+                priv->magic_not_rop_nr = 0x01;
+                break;
+        case 0xc8: /* 4/4/3/4, 5 */
+                priv->magic_not_rop_nr = 0x06;
+                break;
+        case 0xce: /* 4/4/0/0, 4 */
+                priv->magic_not_rop_nr = 0x03;
                 break;
         }
 
@@ -763,13 +872,16 @@ nvc0_graph_create(struct drm_device *dev)
                          priv->tp_nr[3], priv->rop_nr);
                 /* use 0xc3's values... */
                 priv->magic_not_rop_nr = 0x03;
-                priv->magicgpc918      = 0x00200000;
         }
 
         NVOBJ_CLASS(dev, 0x902d, GR); /* 2D */
         NVOBJ_CLASS(dev, 0x9039, GR); /* M2MF */
         NVOBJ_MTHD (dev, 0x9039, 0x0500, nvc0_graph_mthd_page_flip);
         NVOBJ_CLASS(dev, 0x9097, GR); /* 3D */
+        if (fermi >= 0x9197)
+                NVOBJ_CLASS(dev, 0x9197, GR); /* 3D (NVC1-) */
+        if (fermi >= 0x9297)
+                NVOBJ_CLASS(dev, 0x9297, GR); /* 3D (NVC8-) */
         NVOBJ_CLASS(dev, 0x90c0, GR); /* COMPUTE */
         return 0;
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.fuc b/drivers/gpu/drm/nouveau/nvc0_graph.fuc
new file mode 100644
index 00000000000..2a4b6dc8f9d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.fuc
@@ -0,0 +1,400 @@
+/* fuc microcode util functions for nvc0 PGRAPH
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+define(`mmctx_data', `.b32 eval((($2 - 1) << 26) | $1)')
+define(`queue_init', `.skip eval((2 * 4) + ((8 * 4) * 2))')
+
+ifdef(`include_code', `
+// Error codes
+define(`E_BAD_COMMAND', 0x01)
+define(`E_CMD_OVERFLOW', 0x02)
+
+// Util macros to help with debugging ucode hangs etc
+define(`T_WAIT', 0)
+define(`T_MMCTX', 1)
+define(`T_STRWAIT', 2)
+define(`T_STRINIT', 3)
+define(`T_AUTO', 4)
+define(`T_CHAN', 5)
+define(`T_LOAD', 6)
+define(`T_SAVE', 7)
+define(`T_LCHAN', 8)
+define(`T_LCTXH', 9)
+
+define(`trace_set', `
+        mov $r8 0x83c
+        shl b32 $r8 6
+        clear b32 $r9
+        bset $r9 $1
+        iowr I[$r8 + 0x000] $r9         // CC_SCRATCH[7]
+')
+
+define(`trace_clr', `
+        mov $r8 0x85c
+        shl b32 $r8 6
+        clear b32 $r9
+        bset $r9 $1
+        iowr I[$r8 + 0x000] $r9         // CC_SCRATCH[7]
+')
+
+// queue_put - add request to queue
+//
+// In : $r13 queue pointer
+//      $r14 command
+//      $r15 data
+//
+queue_put:
+        // make sure we have space..
+        ld b32 $r8 D[$r13 + 0x0]        // GET
+        ld b32 $r9 D[$r13 + 0x4]        // PUT
+        xor $r8 8
+        cmpu b32 $r8 $r9
+        bra ne queue_put_next
+                mov $r15 E_CMD_OVERFLOW
+                call error
+                ret
+
+        // store cmd/data on queue
+        queue_put_next:
+        and $r8 $r9 7
+        shl b32 $r8 3
+        add b32 $r8 $r13
+        add b32 $r8 8
+        st b32 D[$r8 + 0x0] $r14
+        st b32 D[$r8 + 0x4] $r15
+
+        // update PUT
+        add b32 $r9 1
+        and $r9 0xf
+        st b32 D[$r13 + 0x4] $r9
+        ret
+
+// queue_get - fetch request from queue
+//
+// In : $r13 queue pointer
+//
+// Out: $p1  clear on success (data available)
+//      $r14 command
+//      $r15 data
+//
+queue_get:
+        bset $flags $p1
+        ld b32 $r8 D[$r13 + 0x0]        // GET
+        ld b32 $r9 D[$r13 + 0x4]        // PUT
+        cmpu b32 $r8 $r9
+        bra e queue_get_done
+                // fetch first cmd/data pair
+                and $r9 $r8 7
+                shl b32 $r9 3
+                add b32 $r9 $r13
+                add b32 $r9 8
+                ld b32 $r14 D[$r9 + 0x0]
+                ld b32 $r15 D[$r9 + 0x4]
+
+                // update GET
+                add b32 $r8 1
+                and $r8 0xf
+                st b32 D[$r13 + 0x0] $r8
+                bclr $flags $p1
+queue_get_done:
+        ret
+
+// nv_rd32 - read 32-bit value from nv register
+//
+// In : $r14 register
+// Out: $r15 value
+//
+nv_rd32:
+        mov $r11 0x728
+        shl b32 $r11 6
+        mov b32 $r12 $r14
+        bset $r12 31                    // MMIO_CTRL_PENDING
+        iowr I[$r11 + 0x000] $r12       // MMIO_CTRL
+        nv_rd32_wait:
+                iord $r12 I[$r11 + 0x000]
+                xbit $r12 $r12 31
+                bra ne nv_rd32_wait
+        mov $r10 6                      // DONE_MMIO_RD
+        call wait_doneo
+        iord $r15 I[$r11 + 0x100]       // MMIO_RDVAL
+        ret
+
+// nv_wr32 - write 32-bit value to nv register
+//
+// In : $r14 register
+//      $r15 value
+//
+nv_wr32:
+        mov $r11 0x728
+        shl b32 $r11 6
+        iowr I[$r11 + 0x200] $r15       // MMIO_WRVAL
+        mov b32 $r12 $r14
+        bset $r12 31                    // MMIO_CTRL_PENDING
+        bset $r12 30                    // MMIO_CTRL_WRITE
+        iowr I[$r11 + 0x000] $r12       // MMIO_CTRL
+        nv_wr32_wait:
+                iord $r12 I[$r11 + 0x000]
+                xbit $r12 $r12 31
+                bra ne nv_wr32_wait
+        ret
+
+// (re)set watchdog timer
+//
+// In : $r15 timeout
+//
+watchdog_reset:
+        mov $r8 0x430
+        shl b32 $r8 6
+        bset $r15 31
+        iowr I[$r8 + 0x000] $r15
+        ret
+
+// clear watchdog timer
+watchdog_clear:
+        mov $r8 0x430
+        shl b32 $r8 6
+        iowr I[$r8 + 0x000] $r0
+        ret
+
+// wait_done{z,o} - wait on FUC_DONE bit to become clear/set
+//
+// In : $r10 bit to wait on
+//
+define(`wait_done', `
+$1:
+        trace_set(T_WAIT);
+        mov $r8 0x818
+        shl b32 $r8 6
+        iowr I[$r8 + 0x000] $r10        // CC_SCRATCH[6] = wait bit
+        wait_done_$1:
+                mov $r8 0x400
+                shl b32 $r8 6
+                iord $r8 I[$r8 + 0x000] // DONE
+                xbit $r8 $r8 $r10
+                bra $2 wait_done_$1
+        trace_clr(T_WAIT)
+        ret
+')
+wait_done(wait_donez, ne)
+wait_done(wait_doneo, e)
+
+// mmctx_size - determine size of a mmio list transfer
+//
+// In : $r14 mmio list head
+//      $r15 mmio list tail
+// Out: $r15 transfer size (in bytes)
+//
+mmctx_size:
+        clear b32 $r9
+        nv_mmctx_size_loop:
+                ld b32 $r8 D[$r14]
+                shr b32 $r8 26
+                add b32 $r8 1
+                shl b32 $r8 2
+                add b32 $r9 $r8
+                add b32 $r14 4
+                cmpu b32 $r14 $r15
+                bra ne nv_mmctx_size_loop
+        mov b32 $r15 $r9
+        ret
+
+// mmctx_xfer - execute a list of mmio transfers
+//
+// In : $r10 flags
+//              bit 0: direction (0 = save, 1 = load)
+//              bit 1: set if first transfer
+//              bit 2: set if last transfer
+//      $r11 base
+//      $r12 mmio list head
+//      $r13 mmio list tail
+//      $r14 multi_stride
+//      $r15 multi_mask
+//
+mmctx_xfer:
+        trace_set(T_MMCTX)
+        mov $r8 0x710
+        shl b32 $r8 6
+        clear b32 $r9
+        or $r11 $r11
+        bra e mmctx_base_disabled
+                iowr I[$r8 + 0x000] $r11        // MMCTX_BASE
+                bset $r9 0                      // BASE_EN
+        mmctx_base_disabled:
+        or $r14 $r14
+        bra e mmctx_multi_disabled
+                iowr I[$r8 + 0x200] $r14        // MMCTX_MULTI_STRIDE
+                iowr I[$r8 + 0x300] $r15        // MMCTX_MULTI_MASK
+                bset $r9 1                      // MULTI_EN
+        mmctx_multi_disabled:
+        add b32 $r8 0x100
+
+        xbit $r11 $r10 0
+        shl b32 $r11 16                 // DIR
+        bset $r11 12                    // QLIMIT = 0x10
+        xbit $r14 $r10 1
+        shl b32 $r14 17
+        or $r11 $r14                    // START_TRIGGER
+        iowr I[$r8 + 0x000] $r11        // MMCTX_CTRL
+
+        // loop over the mmio list, and send requests to the hw
+        mmctx_exec_loop:
+                // wait for space in mmctx queue
+                mmctx_wait_free:
+                        iord $r14 I[$r8 + 0x000] // MMCTX_CTRL
+                        and $r14 0x1f
+                        bra e mmctx_wait_free
+
+                // queue up an entry
+                ld b32 $r14 D[$r12]
+                or $r14 $r9
+                iowr I[$r8 + 0x300] $r14
+                add b32 $r12 4
+                cmpu b32 $r12 $r13
+                bra ne mmctx_exec_loop
+
+        xbit $r11 $r10 2
+        bra ne mmctx_stop
+                // wait for queue to empty
+                mmctx_fini_wait:
+                        iord $r11 I[$r8 + 0x000]        // MMCTX_CTRL
+                        and $r11 0x1f
+                        cmpu b32 $r11 0x10
+                        bra ne mmctx_fini_wait
+                mov $r10 2                              // DONE_MMCTX
+                call wait_donez
+                bra mmctx_done
+        mmctx_stop:
+                xbit $r11 $r10 0
+                shl b32 $r11 16                 // DIR
+                bset $r11 12                    // QLIMIT = 0x10
+                bset $r11 18                    // STOP_TRIGGER
+                iowr I[$r8 + 0x000] $r11        // MMCTX_CTRL
+                mmctx_stop_wait:
+                        // wait for STOP_TRIGGER to clear
+                        iord $r11 I[$r8 + 0x000] // MMCTX_CTRL
+                        xbit $r11 $r11 18
+                        bra ne mmctx_stop_wait
+        mmctx_done:
+        trace_clr(T_MMCTX)
+        ret
+
+// Wait for DONE_STRAND
+//
+strand_wait:
+        push $r10
+        mov $r10 2
+        call wait_donez
+        pop $r10
+        ret
+
+// unknown - call before issuing strand commands
+//
+strand_pre:
+        mov $r8 0x4afc
+        sethi $r8 0x20000
+        mov $r9 0xc
+        iowr I[$r8] $r9
+        call strand_wait
+        ret
+
+// unknown - call after issuing strand commands
+//
+strand_post:
+        mov $r8 0x4afc
+        sethi $r8 0x20000
+        mov $r9 0xd
+        iowr I[$r8] $r9
+        call strand_wait
+        ret
+
+// Selects strand set?!
+//
+// In: $r14 id
+//
+strand_set:
+        mov $r10 0x4ffc
+        sethi $r10 0x20000
+        sub b32 $r11 $r10 0x500
+        mov $r12 0xf
+        iowr I[$r10 + 0x000] $r12               // 0x93c = 0xf
+        mov $r12 0xb
+        iowr I[$r11 + 0x000] $r12               // 0x928 = 0xb
+        call strand_wait
+        iowr I[$r10 + 0x000] $r14               // 0x93c = <id>
+        mov $r12 0xa
+        iowr I[$r11 + 0x000] $r12               // 0x928 = 0xa
+        call strand_wait
+        ret
+
+// Initialise strand context data
+//
+// In : $r15 context base
+// Out: $r15 context size (in bytes)
+//
+// Strandset(?) 3 hardcoded currently
+//
+strand_ctx_init:
+        trace_set(T_STRINIT)
+        call strand_pre
+        mov $r14 3
+        call strand_set
+        mov $r10 0x46fc
+        sethi $r10 0x20000
+        add b32 $r11 $r10 0x400
+        iowr I[$r10 + 0x100] $r0        // STRAND_FIRST_GENE = 0
+        mov $r12 1
+        iowr I[$r11 + 0x000] $r12       // STRAND_CMD = LATCH_FIRST_GENE
+        call strand_wait
+        sub b32 $r12 $r0 1
+        iowr I[$r10 + 0x000] $r12       // STRAND_GENE_CNT = 0xffffffff
+        mov $r12 2
+        iowr I[$r11 + 0x000] $r12       // STRAND_CMD = LATCH_GENE_CNT
+        call strand_wait
+        call strand_post
+
+        // read the size of each strand, poke the context offset of
+        // each into STRAND_{SAVE,LOAD}_SWBASE now, no need to worry
+        // about it later then.
+        mov $r8 0x880
+        shl b32 $r8 6
+        iord $r9 I[$r8 + 0x000]         // STRANDS
+        add b32 $r8 0x2200
+        shr b32 $r14 $r15 8
+        ctx_init_strand_loop:
+                iowr I[$r8 + 0x000] $r14        // STRAND_SAVE_SWBASE
+                iowr I[$r8 + 0x100] $r14        // STRAND_LOAD_SWBASE
+                iord $r10 I[$r8 + 0x200]        // STRAND_SIZE
+                shr b32 $r10 6
+                add b32 $r10 1
+                add b32 $r14 $r10
+                add b32 $r8 4
+                sub b32 $r9 1
+                bra ne ctx_init_strand_loop
+
+        shl b32 $r14 8
+        sub b32 $r15 $r14 $r15
+        trace_clr(T_STRINIT)
+        ret
+')
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.h b/drivers/gpu/drm/nouveau/nvc0_graph.h
index f5d184e0689..55689e99728 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.h
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.h
@@ -57,8 +57,7 @@ struct nvc0_graph_priv {
         struct nouveau_gpuobj *unk4188b4;
         struct nouveau_gpuobj *unk4188b8;
 
-        u8  magic_not_rop_nr;
-        u32 magicgpc918;
+        u8 magic_not_rop_nr;
 };
 
 struct nvc0_graph_chan {
@@ -72,4 +71,25 @@ struct nvc0_graph_chan {
 
 int nvc0_grctx_generate(struct nouveau_channel *);
 
+/* nvc0_graph.c uses this also to determine supported chipsets */
+static inline u32
+nvc0_graph_class(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        switch (dev_priv->chipset) {
+        case 0xc0:
+        case 0xc3:
+        case 0xc4:
+        case 0xce: /* guess, mmio trace shows only 0x9097 state */
+                return 0x9097;
+        case 0xc1:
+                return 0x9197;
+        case 0xc8:
+                return 0x9297;
+        default:
+                return 0;
+        }
+}
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvc0_grctx.c b/drivers/gpu/drm/nouveau/nvc0_grctx.c
index 6df06611413..31018eaf527 100644
--- a/drivers/gpu/drm/nouveau/nvc0_grctx.c
+++ b/drivers/gpu/drm/nouveau/nvc0_grctx.c
@@ -45,6 +45,9 @@ nv_mthd(struct drm_device *dev, u32 class, u32 mthd, u32 data)
 static void
 nvc0_grctx_generate_9097(struct drm_device *dev)
 {
+        u32 fermi = nvc0_graph_class(dev);
+        u32 mthd;
+
         nv_mthd(dev, 0x9097, 0x0800, 0x00000000);
         nv_mthd(dev, 0x9097, 0x0840, 0x00000000);
         nv_mthd(dev, 0x9097, 0x0880, 0x00000000);
@@ -824,134 +827,10 @@ nvc0_grctx_generate_9097(struct drm_device *dev)
         nv_mthd(dev, 0x9097, 0x1eb8, 0x00000001);
         nv_mthd(dev, 0x9097, 0x1ed8, 0x00000001);
         nv_mthd(dev, 0x9097, 0x1ef8, 0x00000001);
-        nv_mthd(dev, 0x9097, 0x3400, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3404, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3408, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x340c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3410, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3414, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3418, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x341c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3420, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3424, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3428, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x342c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3430, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3434, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3438, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x343c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3440, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3444, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3448, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x344c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3450, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3454, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3458, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x345c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3460, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3464, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3468, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x346c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3470, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3474, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3478, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x347c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3480, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3484, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3488, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x348c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3490, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3494, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3498, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x349c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34a0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34a4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34a8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34ac, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34b0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34b4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34b8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34bc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34c0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34c4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34c8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34cc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34d0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34d4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34d8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34dc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34e0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34e4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34e8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34ec, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34f0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34f4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34f8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34fc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3500, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3504, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3508, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x350c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3510, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3514, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3518, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x351c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3520, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3524, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3528, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x352c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3530, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3534, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3538, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x353c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3540, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3544, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3548, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x354c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3550, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3554, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3558, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x355c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3560, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3564, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3568, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x356c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3570, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3574, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3578, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x357c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3580, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3584, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3588, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x358c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3590, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3594, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3598, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x359c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35a0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35a4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35a8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35ac, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35b0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35b4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35b8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35bc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35c0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35c4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35c8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35cc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35d0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35d4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35d8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35dc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35e0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35e4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35e8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35ec, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35f0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35f4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35f8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35fc, 0x00000000);
+        if (fermi == 0x9097) {
+                for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+                        nv_mthd(dev, 0x9097, mthd, 0x00000000);
+        }
         nv_mthd(dev, 0x9097, 0x030c, 0x00000001);
         nv_mthd(dev, 0x9097, 0x1944, 0x00000000);
         nv_mthd(dev, 0x9097, 0x1514, 0x00000000);
@@ -1321,6 +1200,37 @@ nvc0_grctx_generate_9097(struct drm_device *dev)
 }
 
 static void
+nvc0_grctx_generate_9197(struct drm_device *dev)
+{
+        u32 fermi = nvc0_graph_class(dev);
+        u32 mthd;
+
+        if (fermi == 0x9197) {
+                for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+                        nv_mthd(dev, 0x9197, mthd, 0x00000000);
+        }
+        nv_mthd(dev, 0x9197, 0x02e4, 0x0000b001);
+}
+
+static void
+nvc0_grctx_generate_9297(struct drm_device *dev)
+{
+        u32 fermi = nvc0_graph_class(dev);
+        u32 mthd;
+
+        if (fermi == 0x9297) {
+                for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+                        nv_mthd(dev, 0x9297, mthd, 0x00000000);
+        }
+        nv_mthd(dev, 0x9297, 0x036c, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x0370, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x07a4, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x07a8, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x0374, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x0378, 0x00000020);
+}
+
+static void
 nvc0_grctx_generate_902d(struct drm_device *dev)
 {
         nv_mthd(dev, 0x902d, 0x0200, 0x000000cf);
@@ -1559,8 +1469,15 @@ nvc0_grctx_generate_unk47xx(struct drm_device *dev)
 static void
 nvc0_grctx_generate_shaders(struct drm_device *dev)
 {
-        nv_wr32(dev, 0x405800, 0x078000bf);
-        nv_wr32(dev, 0x405830, 0x02180000);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->chipset != 0xc1) {
+                nv_wr32(dev, 0x405800, 0x078000bf);
+                nv_wr32(dev, 0x405830, 0x02180000);
+        } else {
+                nv_wr32(dev, 0x405800, 0x0f8000bf);
+                nv_wr32(dev, 0x405830, 0x02180218);
+        }
         nv_wr32(dev, 0x405834, 0x00000000);
         nv_wr32(dev, 0x405838, 0x00000000);
         nv_wr32(dev, 0x405854, 0x00000000);
@@ -1586,10 +1503,16 @@ nvc0_grctx_generate_unk60xx(struct drm_device *dev)
 static void
 nvc0_grctx_generate_unk64xx(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+
         nv_wr32(dev, 0x4064a8, 0x00000000);
         nv_wr32(dev, 0x4064ac, 0x00003fff);
         nv_wr32(dev, 0x4064b4, 0x00000000);
         nv_wr32(dev, 0x4064b8, 0x00000000);
+        if (dev_priv->chipset == 0xc1) {
+                nv_wr32(dev, 0x4064c0, 0x80140078);
+                nv_wr32(dev, 0x4064c4, 0x0086ffff);
+        }
 }
 
 static void
@@ -1622,21 +1545,14 @@ static void
 nvc0_grctx_generate_rop(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int chipset = dev_priv->chipset;
 
         /* ROPC_BROADCAST */
         nv_wr32(dev, 0x408800, 0x02802a3c);
         nv_wr32(dev, 0x408804, 0x00000040);
-        nv_wr32(dev, 0x408808, 0x0003e00d);
-        switch (dev_priv->chipset) {
-        case 0xc0:
-                nv_wr32(dev, 0x408900, 0x0080b801);
-                break;
-        case 0xc3:
-        case 0xc4:
-                nv_wr32(dev, 0x408900, 0x3080b801);
-                break;
-        }
-        nv_wr32(dev, 0x408904, 0x02000001);
+        nv_wr32(dev, 0x408808, chipset != 0xc1 ? 0x0003e00d : 0x1003e005);
+        nv_wr32(dev, 0x408900, 0x3080b801);
+        nv_wr32(dev, 0x408904, chipset != 0xc1 ? 0x02000001 : 0x62000001);
         nv_wr32(dev, 0x408908, 0x00c80929);
         nv_wr32(dev, 0x40890c, 0x00000000);
         nv_wr32(dev, 0x408980, 0x0000011d);
@@ -1645,6 +1561,8 @@ nvc0_grctx_generate_rop(struct drm_device *dev)
 static void
 nvc0_grctx_generate_gpc(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int chipset = dev_priv->chipset;
         int i;
 
         /* GPC_BROADCAST */
@@ -1676,7 +1594,7 @@ nvc0_grctx_generate_gpc(struct drm_device *dev)
         nv_wr32(dev, 0x41880c, 0x00000000);
         nv_wr32(dev, 0x418810, 0x00000000);
         nv_wr32(dev, 0x418828, 0x00008442);
-        nv_wr32(dev, 0x418830, 0x00000001);
+        nv_wr32(dev, 0x418830, chipset != 0xc1 ? 0x00000001 : 0x10000001);
         nv_wr32(dev, 0x4188d8, 0x00000008);
         nv_wr32(dev, 0x4188e0, 0x01000000);
         nv_wr32(dev, 0x4188e8, 0x00000000);
@@ -1684,7 +1602,7 @@ nvc0_grctx_generate_gpc(struct drm_device *dev)
         nv_wr32(dev, 0x4188f0, 0x00000000);
         nv_wr32(dev, 0x4188f4, 0x00000000);
         nv_wr32(dev, 0x4188f8, 0x00000000);
-        nv_wr32(dev, 0x4188fc, 0x00100000);
+        nv_wr32(dev, 0x4188fc, chipset != 0xc1 ? 0x00100000 : 0x00100018);
         nv_wr32(dev, 0x41891c, 0x00ff00ff);
         nv_wr32(dev, 0x418924, 0x00000000);
         nv_wr32(dev, 0x418928, 0x00ffff00);
@@ -1715,6 +1633,8 @@ nvc0_grctx_generate_gpc(struct drm_device *dev)
         nv_wr32(dev, 0x418c24, 0x00000000);
         nv_wr32(dev, 0x418c28, 0x00000000);
         nv_wr32(dev, 0x418c2c, 0x00000000);
+        if (chipset == 0xc1)
+                nv_wr32(dev, 0x418c6c, 0x00000001);
         nv_wr32(dev, 0x418c80, 0x20200004);
         nv_wr32(dev, 0x418c8c, 0x00000001);
         nv_wr32(dev, 0x419000, 0x00000780);
@@ -1727,10 +1647,13 @@ static void
 nvc0_grctx_generate_tp(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int chipset = dev_priv->chipset;
 
         /* GPC_BROADCAST.TP_BROADCAST */
+        nv_wr32(dev, 0x419818, 0x00000000);
+        nv_wr32(dev, 0x41983c, 0x00038bc7);
         nv_wr32(dev, 0x419848, 0x00000000);
-        nv_wr32(dev, 0x419864, 0x0000012a);
+        nv_wr32(dev, 0x419864, chipset != 0xc1 ? 0x0000012a : 0x00000129);
         nv_wr32(dev, 0x419888, 0x00000000);
         nv_wr32(dev, 0x419a00, 0x000001f0);
         nv_wr32(dev, 0x419a04, 0x00000001);
@@ -1740,8 +1663,8 @@ nvc0_grctx_generate_tp(struct drm_device *dev)
         nv_wr32(dev, 0x419a14, 0x00000200);
         nv_wr32(dev, 0x419a1c, 0x00000000);
         nv_wr32(dev, 0x419a20, 0x00000800);
-        if (dev_priv->chipset != 0xc0)
-                nv_wr32(dev, 0x00419ac4, 0x0007f440); /* 0xc3 */
+        if (chipset != 0xc0 && chipset != 0xc8)
+                nv_wr32(dev, 0x00419ac4, 0x0007f440);
         nv_wr32(dev, 0x419b00, 0x0a418820);
         nv_wr32(dev, 0x419b04, 0x062080e6);
         nv_wr32(dev, 0x419b08, 0x020398a4);
@@ -1749,17 +1672,19 @@ nvc0_grctx_generate_tp(struct drm_device *dev)
         nv_wr32(dev, 0x419b10, 0x0a418820);
         nv_wr32(dev, 0x419b14, 0x000000e6);
         nv_wr32(dev, 0x419bd0, 0x00900103);
-        nv_wr32(dev, 0x419be0, 0x00000001);
+        nv_wr32(dev, 0x419be0, chipset != 0xc1 ? 0x00000001 : 0x00400001);
         nv_wr32(dev, 0x419be4, 0x00000000);
         nv_wr32(dev, 0x419c00, 0x00000002);
         nv_wr32(dev, 0x419c04, 0x00000006);
         nv_wr32(dev, 0x419c08, 0x00000002);
         nv_wr32(dev, 0x419c20, 0x00000000);
-        nv_wr32(dev, 0x419cbc, 0x28137606);
+        nv_wr32(dev, 0x419cb0, 0x00060048); //XXX: 0xce 0x00020048
         nv_wr32(dev, 0x419ce8, 0x00000000);
         nv_wr32(dev, 0x419cf4, 0x00000183);
-        nv_wr32(dev, 0x419d20, 0x02180000);
+        nv_wr32(dev, 0x419d20, chipset != 0xc1 ? 0x02180000 : 0x12180000);
         nv_wr32(dev, 0x419d24, 0x00001fff);
+        if (chipset == 0xc1)
+                nv_wr32(dev, 0x419d44, 0x02180218);
         nv_wr32(dev, 0x419e04, 0x00000000);
         nv_wr32(dev, 0x419e08, 0x00000000);
         nv_wr32(dev, 0x419e0c, 0x00000000);
@@ -1785,11 +1710,11 @@ nvc0_grctx_generate_tp(struct drm_device *dev)
         nv_wr32(dev, 0x419e8c, 0x00000000);
         nv_wr32(dev, 0x419e90, 0x00000000);
         nv_wr32(dev, 0x419e98, 0x00000000);
-        if (dev_priv->chipset != 0xc0)
+        if (chipset != 0xc0 && chipset != 0xc8)
                 nv_wr32(dev, 0x419ee0, 0x00011110);
         nv_wr32(dev, 0x419f50, 0x00000000);
         nv_wr32(dev, 0x419f54, 0x00000000);
-        if (dev_priv->chipset != 0xc0)
+        if (chipset != 0xc0 && chipset != 0xc8)
                 nv_wr32(dev, 0x419f58, 0x00000000);
 }
 
@@ -1801,6 +1726,7 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
         struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
         struct drm_device *dev = chan->dev;
         int i, gpc, tp, id;
+        u32 fermi = nvc0_graph_class(dev);
         u32 r000260, tmp;
 
         r000260 = nv_rd32(dev, 0x000260);
@@ -1857,10 +1783,11 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
         nv_wr32(dev, 0x40587c, 0x00000000);
 
         if (1) {
-                const u8 chipset_tp_max[] = { 16, 0, 0, 4, 8 };
+                const u8 chipset_tp_max[] = { 16, 4, 0, 4, 8, 0, 0, 0,
+                                              16, 0, 0, 0, 0, 0, 8, 0 };
                 u8 max = chipset_tp_max[dev_priv->chipset & 0x0f];
                 u8 tpnr[GPC_MAX];
-                u8 data[32];
+                u8 data[TP_MAX];
 
                 memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
                 memset(data, 0x1f, sizeof(data));
@@ -2633,6 +2560,8 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
         nv_icmd(dev, 0x0000053f, 0xffff0000);
         nv_icmd(dev, 0x00000585, 0x0000003f);
         nv_icmd(dev, 0x00000576, 0x00000003);
+        if (dev_priv->chipset == 0xc1)
+                nv_icmd(dev, 0x0000057b, 0x00000059);
         nv_icmd(dev, 0x00000586, 0x00000040);
         nv_icmd(dev, 0x00000582, 0x00000080);
         nv_icmd(dev, 0x00000583, 0x00000080);
@@ -2865,6 +2794,10 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
         nv_wr32(dev, 0x404154, 0x00000400);
 
         nvc0_grctx_generate_9097(dev);
+        if (fermi >= 0x9197)
+                nvc0_grctx_generate_9197(dev);
+        if (fermi >= 0x9297)
+                nvc0_grctx_generate_9297(dev);
         nvc0_grctx_generate_902d(dev);
         nvc0_grctx_generate_9039(dev);
         nvc0_grctx_generate_90c0(dev);
diff --git a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc
new file mode 100644
index 00000000000..0ec2add72a7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc
@@ -0,0 +1,474 @@
+/* fuc microcode for nvc0 PGRAPH/GPC
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+/* To build:
+ *    m4 nvc0_grgpc.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grgpc.fuc.h
+ */
+
+/* TODO
+ * - bracket certain functions with scratch writes, useful for debugging
+ * - watchdog timer around ctx operations
+ */
+
+.section nvc0_grgpc_data
+include(`nvc0_graph.fuc')
+gpc_id:                 .b32 0
+gpc_mmio_list_head:     .b32 0
+gpc_mmio_list_tail:     .b32 0
+
+tpc_count:              .b32 0
+tpc_mask:               .b32 0
+tpc_mmio_list_head:     .b32 0
+tpc_mmio_list_tail:     .b32 0
+
+cmd_queue:              queue_init
+
+// chipset descriptions
+chipsets:
+.b8  0xc0 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc0_tpc_mmio_tail
+.b8  0xc1 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc1_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc1_tpc_mmio_tail
+.b8  0xc3 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc3_tpc_mmio_tail
+.b8  0xc4 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc3_tpc_mmio_tail
+.b8  0xc8 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc0_tpc_mmio_tail
+.b8  0xce 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc3_tpc_mmio_tail
+.b8  0 0 0 0
+
+// GPC mmio lists
+nvc0_gpc_mmio_head:
+mmctx_data(0x000380, 1)
+mmctx_data(0x000400, 6)
+mmctx_data(0x000450, 9)
+mmctx_data(0x000600, 1)
+mmctx_data(0x000684, 1)
+mmctx_data(0x000700, 5)
+mmctx_data(0x000800, 1)
+mmctx_data(0x000808, 3)
+mmctx_data(0x000828, 1)
+mmctx_data(0x000830, 1)
+mmctx_data(0x0008d8, 1)
+mmctx_data(0x0008e0, 1)
+mmctx_data(0x0008e8, 6)
+mmctx_data(0x00091c, 1)
+mmctx_data(0x000924, 3)
+mmctx_data(0x000b00, 1)
+mmctx_data(0x000b08, 6)
+mmctx_data(0x000bb8, 1)
+mmctx_data(0x000c08, 1)
+mmctx_data(0x000c10, 8)
+mmctx_data(0x000c80, 1)
+mmctx_data(0x000c8c, 1)
+mmctx_data(0x001000, 3)
+mmctx_data(0x001014, 1)
+nvc0_gpc_mmio_tail:
+mmctx_data(0x000c6c, 1);
+nvc1_gpc_mmio_tail:
+
+// TPC mmio lists
+nvc0_tpc_mmio_head:
+mmctx_data(0x000018, 1)
+mmctx_data(0x00003c, 1)
+mmctx_data(0x000048, 1)
+mmctx_data(0x000064, 1)
+mmctx_data(0x000088, 1)
+mmctx_data(0x000200, 6)
+mmctx_data(0x00021c, 2)
+mmctx_data(0x000300, 6)
+mmctx_data(0x0003d0, 1)
+mmctx_data(0x0003e0, 2)
+mmctx_data(0x000400, 3)
+mmctx_data(0x000420, 1)
+mmctx_data(0x0004b0, 1)
+mmctx_data(0x0004e8, 1)
+mmctx_data(0x0004f4, 1)
+mmctx_data(0x000520, 2)
+mmctx_data(0x000604, 4)
+mmctx_data(0x000644, 20)
+mmctx_data(0x000698, 1)
+mmctx_data(0x000750, 2)
+nvc0_tpc_mmio_tail:
+mmctx_data(0x000758, 1)
+mmctx_data(0x0002c4, 1)
+mmctx_data(0x0004bc, 1)
+mmctx_data(0x0006e0, 1)
+nvc3_tpc_mmio_tail:
+mmctx_data(0x000544, 1)
+nvc1_tpc_mmio_tail:
+
+
+.section nvc0_grgpc_code
+bra init
+define(`include_code')
+include(`nvc0_graph.fuc')
+
+// reports an exception to the host
+//
+// In: $r15 error code (see nvc0_graph.fuc)
+//
+error:
+        push $r14
+        mov $r14 -0x67ec        // 0x9814
+        sethi $r14 0x400000
+        call nv_wr32            // HUB_CTXCTL_CC_SCRATCH[5] = error code
+        add b32 $r14 0x41c
+        mov $r15 1
+        call nv_wr32            // HUB_CTXCTL_INTR_UP_SET
+        pop $r14
+        ret
+
+// GPC fuc initialisation, executed by triggering ucode start, will
+// fall through to main loop after completion.
+//
+// Input:
+//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
+//   CC_SCRATCH[1]: context base
+//
+// Output:
+//   CC_SCRATCH[0]:
+//           31:31: set to signal completion
+//   CC_SCRATCH[1]:
+//            31:0: GPC context size
+//
+init:
+        clear b32 $r0
+        mov $sp $r0
+
+        // enable fifo access
+        mov $r1 0x1200
+        mov $r2 2
+        iowr I[$r1 + 0x000] $r2         // FIFO_ENABLE
+
+        // setup i0 handler, and route all interrupts to it
+        mov $r1 ih
+        mov $iv0 $r1
+        mov $r1 0x400
+        iowr I[$r1 + 0x300] $r0         // INTR_DISPATCH
+
+        // enable fifo interrupt
+        mov $r2 4
+        iowr I[$r1 + 0x000] $r2         // INTR_EN_SET
+
+        // enable interrupts
+        bset $flags ie0
+
+        // figure out which GPC we are, and how many TPCs we have
+        mov $r1 0x608
+        shl b32 $r1 6
+        iord $r2 I[$r1 + 0x000]         // UNITS
+        mov $r3 1
+        and $r2 0x1f
+        shl b32 $r3 $r2
+        sub b32 $r3 1
+        st b32 D[$r0 + tpc_count] $r2
+        st b32 D[$r0 + tpc_mask] $r3
+        add b32 $r1 0x400
+        iord $r2 I[$r1 + 0x000]         // MYINDEX
+        st b32 D[$r0 + gpc_id] $r2
+
+        // find context data for this chipset
+        mov $r2 0x800
+        shl b32 $r2 6
+        iord $r2 I[$r2 + 0x000]         // CC_SCRATCH[0]
+        mov $r1 chipsets - 12
+        init_find_chipset:
+                add b32 $r1 12
+                ld b32 $r3 D[$r1 + 0x00]
+                cmpu b32 $r3 $r2
+                bra e init_context
+                cmpu b32 $r3 0
+                bra ne init_find_chipset
+                // unknown chipset
+                ret
+
+        // initialise context base, and size tracking
+        init_context:
+        mov $r2 0x800
+        shl b32 $r2 6
+        iord $r2 I[$r2 + 0x100] // CC_SCRATCH[1], initial base
+        clear b32 $r3           // track GPC context size here
+
+        // set mmctx base addresses now so we don't have to do it later,
+        // they don't currently ever change
+        mov $r4 0x700
+        shl b32 $r4 6
+        shr b32 $r5 $r2 8
+        iowr I[$r4 + 0x000] $r5         // MMCTX_SAVE_SWBASE
+        iowr I[$r4 + 0x100] $r5         // MMCTX_LOAD_SWBASE
+
+        // calculate GPC mmio context size, store the chipset-specific
+        // mmio list pointers somewhere we can get at them later without
+        // re-parsing the chipset list
+        clear b32 $r14
+        clear b32 $r15
+        ld b16 $r14 D[$r1 + 4]
+        ld b16 $r15 D[$r1 + 6]
+        st b16 D[$r0 + gpc_mmio_list_head] $r14
+        st b16 D[$r0 + gpc_mmio_list_tail] $r15
+        call mmctx_size
+        add b32 $r2 $r15
+        add b32 $r3 $r15
+
+        // calculate per-TPC mmio context size, store the list pointers
+        ld b16 $r14 D[$r1 + 8]
+        ld b16 $r15 D[$r1 + 10]
+        st b16 D[$r0 + tpc_mmio_list_head] $r14
+        st b16 D[$r0 + tpc_mmio_list_tail] $r15
+        call mmctx_size
+        ld b32 $r14 D[$r0 + tpc_count]
+        mulu $r14 $r15
+        add b32 $r2 $r14
+        add b32 $r3 $r14
+
+        // round up base/size to 256 byte boundary (for strand SWBASE)
+        add b32 $r4 0x1300
+        shr b32 $r3 2
+        iowr I[$r4 + 0x000] $r3         // MMCTX_LOAD_COUNT, wtf for?!?
+        shr b32 $r2 8
+        shr b32 $r3 6
+        add b32 $r2 1
+        add b32 $r3 1
+        shl b32 $r2 8
+        shl b32 $r3 8
+
+        // calculate size of strand context data
+        mov b32 $r15 $r2
+        call strand_ctx_init
+        add b32 $r3 $r15
+
+        // save context size, and tell HUB we're done
+        mov $r1 0x800
+        shl b32 $r1 6
+        iowr I[$r1 + 0x100] $r3         // CC_SCRATCH[1]  = context size
+        add b32 $r1 0x800
+        clear b32 $r2
+        bset $r2 31
+        iowr I[$r1 + 0x000] $r2         // CC_SCRATCH[0] |= 0x80000000
+
+// Main program loop, very simple, sleeps until woken up by the interrupt
+// handler, pulls a command from the queue and executes its handler
+//
+main:
+        bset $flags $p0
+        sleep $p0
+        mov $r13 cmd_queue
+        call queue_get
+        bra $p1 main
+
+        // 0x0000-0x0003 are all context transfers
+        cmpu b32 $r14 0x04
+        bra nc main_not_ctx_xfer
+                // fetch $flags and mask off $p1/$p2
+                mov $r1 $flags
+                mov $r2 0x0006
+                not b32 $r2
+                and $r1 $r2
+                // set $p1/$p2 according to transfer type
+                shl b32 $r14 1
+                or $r1 $r14
+                mov $flags $r1
+                // transfer context data
+                call ctx_xfer
+                bra main
+
+        main_not_ctx_xfer:
+        shl b32 $r15 $r14 16
+        or $r15 E_BAD_COMMAND
+        call error
+        bra main
+
+// interrupt handler
+ih:
+        push $r8
+        mov $r8 $flags
+        push $r8
+        push $r9
+        push $r10
+        push $r11
+        push $r13
+        push $r14
+        push $r15
+
+        // incoming fifo command?
+        iord $r10 I[$r0 + 0x200]        // INTR
+        and $r11 $r10 0x00000004
+        bra e ih_no_fifo
+                // queue incoming fifo command for later processing
+                mov $r11 0x1900
+                mov $r13 cmd_queue
+                iord $r14 I[$r11 + 0x100]       // FIFO_CMD
+                iord $r15 I[$r11 + 0x000]       // FIFO_DATA
+                call queue_put
+                add b32 $r11 0x400
+                mov $r14 1
+                iowr I[$r11 + 0x000] $r14       // FIFO_ACK
+
+        // ack, and wake up main()
+        ih_no_fifo:
+        iowr I[$r0 + 0x100] $r10        // INTR_ACK
+
+        pop $r15
+        pop $r14
+        pop $r13
+        pop $r11
+        pop $r10
+        pop $r9
+        pop $r8
+        mov $flags $r8
+        pop $r8
+        bclr $flags $p0
+        iret
+
+// Set this GPC's bit in HUB_BAR, used to signal completion of various
+// activities to the HUB fuc
+//
+hub_barrier_done:
+        mov $r15 1
+        ld b32 $r14 D[$r0 + gpc_id]
+        shl b32 $r15 $r14
+        mov $r14 -0x6be8        // 0x409418 - HUB_BAR_SET
+        sethi $r14 0x400000
+        call nv_wr32
+        ret
+
+// Disables various things, waits a bit, and re-enables them..
+//
+// Not sure how exactly this helps, perhaps "ENABLE" is not such a
+// good description for the bits we turn off?  Anyways, without this,
+// funny things happen.
+//
+ctx_redswitch:
+        mov $r14 0x614
+        shl b32 $r14 6
+        mov $r15 0x020
+        iowr I[$r14] $r15       // GPC_RED_SWITCH = POWER
+        mov $r15 8
+        ctx_redswitch_delay:
+                sub b32 $r15 1
+                bra ne ctx_redswitch_delay
+        mov $r15 0xa20
+        iowr I[$r14] $r15       // GPC_RED_SWITCH = UNK11, ENABLE, POWER
+        ret
+
+// Transfer GPC context data between GPU and storage area
+//
+// In: $r15 context base address
+//     $p1 clear on save, set on load
+//     $p2 set if opposite direction done/will be done, so:
+//              on save it means: "a load will follow this save"
+//              on load it means: "a save preceeded this load"
+//
+ctx_xfer:
+        // set context base address
+        mov $r1 0xa04
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r15// MEM_BASE
+        bra not $p1 ctx_xfer_not_load
+                call ctx_redswitch
+        ctx_xfer_not_load:
+
+        // strands
+        mov $r1 0x4afc
+        sethi $r1 0x20000
+        mov $r2 0xc
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x0c
+        call strand_wait
+        mov $r2 0x47fc
+        sethi $r2 0x20000
+        iowr I[$r2] $r0         // STRAND_FIRST_GENE(0x3f) = 0x00
+        xbit $r2 $flags $p1
+        add b32 $r2 3
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)
+
+        // mmio context
+        xbit $r10 $flags $p1    // direction
+        or $r10 2               // first
+        mov $r11 0x0000
+        sethi $r11 0x500000
+        ld b32 $r12 D[$r0 + gpc_id]
+        shl b32 $r12 15
+        add b32 $r11 $r12       // base = NV_PGRAPH_GPCn
+        ld b32 $r12 D[$r0 + gpc_mmio_list_head]
+        ld b32 $r13 D[$r0 + gpc_mmio_list_tail]
+        mov $r14 0              // not multi
+        call mmctx_xfer
+
+        // per-TPC mmio context
+        xbit $r10 $flags $p1    // direction
+        or $r10 4               // last
+        mov $r11 0x4000
+        sethi $r11 0x500000     // base = NV_PGRAPH_GPC0_TPC0
+        ld b32 $r12 D[$r0 + gpc_id]
+        shl b32 $r12 15
+        add b32 $r11 $r12       // base = NV_PGRAPH_GPCn_TPC0
+        ld b32 $r12 D[$r0 + tpc_mmio_list_head]
+        ld b32 $r13 D[$r0 + tpc_mmio_list_tail]
+        ld b32 $r15 D[$r0 + tpc_mask]
+        mov $r14 0x800          // stride = 0x800
+        call mmctx_xfer
+
+        // wait for strands to finish
+        call strand_wait
+
+        // if load, or a save without a load following, do some
+        // unknown stuff that's done after finishing a block of
+        // strand commands
+        bra $p1 ctx_xfer_post
+        bra not $p2 ctx_xfer_done
+        ctx_xfer_post:
+                mov $r1 0x4afc
+                sethi $r1 0x20000
+                mov $r2 0xd
+                iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x0d
+                call strand_wait
+
+        // mark completion in HUB's barrier
+        ctx_xfer_done:
+        call hub_barrier_done
+        ret
+
+.align 256
diff --git a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h
new file mode 100644
index 00000000000..1896c898f5b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h
@@ -0,0 +1,483 @@
+uint32_t nvc0_grgpc_data[] = {
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x000000c0,
+        0x011000b0,
+        0x01640114,
+        0x000000c1,
+        0x011400b0,
+        0x01780114,
+        0x000000c3,
+        0x011000b0,
+        0x01740114,
+        0x000000c4,
+        0x011000b0,
+        0x01740114,
+        0x000000c8,
+        0x011000b0,
+        0x01640114,
+        0x000000ce,
+        0x011000b0,
+        0x01740114,
+        0x00000000,
+        0x00000380,
+        0x14000400,
+        0x20000450,
+        0x00000600,
+        0x00000684,
+        0x10000700,
+        0x00000800,
+        0x08000808,
+        0x00000828,
+        0x00000830,
+        0x000008d8,
+        0x000008e0,
+        0x140008e8,
+        0x0000091c,
+        0x08000924,
+        0x00000b00,
+        0x14000b08,
+        0x00000bb8,
+        0x00000c08,
+        0x1c000c10,
+        0x00000c80,
+        0x00000c8c,
+        0x08001000,
+        0x00001014,
+        0x00000c6c,
+        0x00000018,
+        0x0000003c,
+        0x00000048,
+        0x00000064,
+        0x00000088,
+        0x14000200,
+        0x0400021c,
+        0x14000300,
+        0x000003d0,
+        0x040003e0,
+        0x08000400,
+        0x00000420,
+        0x000004b0,
+        0x000004e8,
+        0x000004f4,
+        0x04000520,
+        0x0c000604,
+        0x4c000644,
+        0x00000698,
+        0x04000750,
+        0x00000758,
+        0x000002c4,
+        0x000004bc,
+        0x000006e0,
+        0x00000544,
+};
+
+uint32_t nvc0_grgpc_code[] = {
+        0x03060ef5,
+        0x9800d898,
+        0x86f001d9,
+        0x0489b808,
+        0xf00c1bf4,
+        0x21f502f7,
+        0x00f802ec,
+        0xb60798c4,
+        0x8dbb0384,
+        0x0880b600,
+        0x80008e80,
+        0x90b6018f,
+        0x0f94f001,
+        0xf801d980,
+        0x0131f400,
+        0x9800d898,
+        0x89b801d9,
+        0x210bf404,
+        0xb60789c4,
+        0x9dbb0394,
+        0x0890b600,
+        0x98009e98,
+        0x80b6019f,
+        0x0f84f001,
+        0xf400d880,
+        0x00f80132,
+        0x0728b7f1,
+        0xb906b4b6,
+        0xc9f002ec,
+        0x00bcd01f,
+        0xc800bccf,
+        0x1bf41fcc,
+        0x06a7f0fa,
+        0x010321f5,
+        0xf840bfcf,
+        0x28b7f100,
+        0x06b4b607,
+        0xb980bfd0,
+        0xc9f002ec,
+        0x1ec9f01f,
+        0xcf00bcd0,
+        0xccc800bc,
+        0xfa1bf41f,
+        0x87f100f8,
+        0x84b60430,
+        0x1ff9f006,
+        0xf8008fd0,
+        0x3087f100,
+        0x0684b604,
+        0xf80080d0,
+        0x3c87f100,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d000,
+        0x081887f1,
+        0xd00684b6,
+        0x87f1008a,
+        0x84b60400,
+        0x0088cf06,
+        0xf4888aff,
+        0x87f1f31b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00099,
+        0xf100f800,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00099f0,
+        0x87f10089,
+        0x84b60818,
+        0x008ad006,
+        0x040087f1,
+        0xcf0684b6,
+        0x8aff0088,
+        0xf30bf488,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0099f094,
+        0xf80089d0,
+        0x9894bd00,
+        0x85b600e8,
+        0x0180b61a,
+        0xbb0284b6,
+        0xe0b60098,
+        0x04efb804,
+        0xb9eb1bf4,
+        0x00f8029f,
+        0x083c87f1,
+        0xbd0684b6,
+        0x0199f094,
+        0xf10089d0,
+        0xb6071087,
+        0x94bd0684,
+        0xf405bbfd,
+        0x8bd0090b,
+        0x0099f000,
+        0xf405eefd,
+        0x8ed00c0b,
+        0xc08fd080,
+        0xb70199f0,
+        0xc8010080,
+        0xb4b600ab,
+        0x0cb9f010,
+        0xb601aec8,
+        0xbefd11e4,
+        0x008bd005,
+        0xf0008ecf,
+        0x0bf41fe4,
+        0x00ce98fa,
+        0xd005e9fd,
+        0xc0b6c08e,
+        0x04cdb804,
+        0xc8e81bf4,
+        0x1bf402ab,
+        0x008bcf18,
+        0xb01fb4f0,
+        0x1bf410b4,
+        0x02a7f0f7,
+        0xf4c921f4,
+        0xabc81b0e,
+        0x10b4b600,
+        0xf00cb9f0,
+        0x8bd012b9,
+        0x008bcf00,
+        0xf412bbc8,
+        0x87f1fa1b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00199,
+        0xf900f800,
+        0x02a7f0a0,
+        0xfcc921f4,
+        0xf100f8a0,
+        0xf04afc87,
+        0x97f00283,
+        0x0089d00c,
+        0x020721f5,
+        0x87f100f8,
+        0x83f04afc,
+        0x0d97f002,
+        0xf50089d0,
+        0xf8020721,
+        0xfca7f100,
+        0x02a3f04f,
+        0x0500aba2,
+        0xd00fc7f0,
+        0xc7f000ac,
+        0x00bcd00b,
+        0x020721f5,
+        0xf000aed0,
+        0xbcd00ac7,
+        0x0721f500,
+        0xf100f802,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x21f50089,
+        0xe7f00213,
+        0x3921f503,
+        0xfca7f102,
+        0x02a3f046,
+        0x0400aba0,
+        0xf040a0d0,
+        0xbcd001c7,
+        0x0721f500,
+        0x010c9202,
+        0xf000acd0,
+        0xbcd002c7,
+        0x0721f500,
+        0x2621f502,
+        0x8087f102,
+        0x0684b608,
+        0xb70089cf,
+        0x95220080,
+        0x8ed008fe,
+        0x408ed000,
+        0xb6808acf,
+        0xa0b606a5,
+        0x00eabb01,
+        0xb60480b6,
+        0x1bf40192,
+        0x08e4b6e8,
+        0xf1f2efbc,
+        0xb6085c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x00f80089,
+        0xe7f1e0f9,
+        0xe3f09814,
+        0x8d21f440,
+        0x041ce0b7,
+        0xf401f7f0,
+        0xe0fc8d21,
+        0x04bd00f8,
+        0xf10004fe,
+        0xf0120017,
+        0x12d00227,
+        0x3e17f100,
+        0x0010fe04,
+        0x040017f1,
+        0xf0c010d0,
+        0x12d00427,
+        0x1031f400,
+        0x060817f1,
+        0xcf0614b6,
+        0x37f00012,
+        0x1f24f001,
+        0xb60432bb,
+        0x02800132,
+        0x04038003,
+        0x040010b7,
+        0x800012cf,
+        0x27f10002,
+        0x24b60800,
+        0x0022cf06,
+        0xb65817f0,
+        0x13980c10,
+        0x0432b800,
+        0xb00b0bf4,
+        0x1bf40034,
+        0xf100f8f1,
+        0xb6080027,
+        0x22cf0624,
+        0xf134bd40,
+        0xb6070047,
+        0x25950644,
+        0x0045d008,
+        0xbd4045d0,
+        0x58f4bde4,
+        0x1f58021e,
+        0x020e4003,
+        0xf5040f40,
+        0xbb013d21,
+        0x3fbb002f,
+        0x041e5800,
+        0x40051f58,
+        0x0f400a0e,
+        0x3d21f50c,
+        0x030e9801,
+        0xbb00effd,
+        0x3ebb002e,
+        0x0040b700,
+        0x0235b613,
+        0xb60043d0,
+        0x35b60825,
+        0x0120b606,
+        0xb60130b6,
+        0x34b60824,
+        0x022fb908,
+        0x026321f5,
+        0xf1003fbb,
+        0xb6080017,
+        0x13d00614,
+        0x0010b740,
+        0xf024bd08,
+        0x12d01f29,
+        0x0031f400,
+        0xf00028f4,
+        0x21f41cd7,
+        0xf401f439,
+        0xf404e4b0,
+        0x81fe1e18,
+        0x0627f001,
+        0x12fd20bd,
+        0x01e4b604,
+        0xfe051efd,
+        0x21f50018,
+        0x0ef404c3,
+        0x10ef94d3,
+        0xf501f5f0,
+        0xf402ec21,
+        0x80f9c60e,
+        0xf90188fe,
+        0xf990f980,
+        0xf9b0f9a0,
+        0xf9e0f9d0,
+        0x800acff0,
+        0xf404abc4,
+        0xb7f11d0b,
+        0xd7f01900,
+        0x40becf1c,
+        0xf400bfcf,
+        0xb0b70421,
+        0xe7f00400,
+        0x00bed001,
+        0xfc400ad0,
+        0xfce0fcf0,
+        0xfcb0fcd0,
+        0xfc90fca0,
+        0x0088fe80,
+        0x32f480fc,
+        0xf001f800,
+        0x0e9801f7,
+        0x04febb00,
+        0x9418e7f1,
+        0xf440e3f0,
+        0x00f88d21,
+        0x0614e7f1,
+        0xf006e4b6,
+        0xefd020f7,
+        0x08f7f000,
+        0xf401f2b6,
+        0xf7f1fd1b,
+        0xefd00a20,
+        0xf100f800,
+        0xb60a0417,
+        0x1fd00614,
+        0x0711f400,
+        0x04a421f5,
+        0x4afc17f1,
+        0xf00213f0,
+        0x12d00c27,
+        0x0721f500,
+        0xfc27f102,
+        0x0223f047,
+        0xf00020d0,
+        0x20b6012c,
+        0x0012d003,
+        0xf001acf0,
+        0xb7f002a5,
+        0x50b3f000,
+        0xb6000c98,
+        0xbcbb0fc4,
+        0x010c9800,
+        0xf0020d98,
+        0x21f500e7,
+        0xacf0015c,
+        0x04a5f001,
+        0x4000b7f1,
+        0x9850b3f0,
+        0xc4b6000c,
+        0x00bcbb0f,
+        0x98050c98,
+        0x0f98060d,
+        0x00e7f104,
+        0x5c21f508,
+        0x0721f501,
+        0x0601f402,
+        0xf11412f4,
+        0xf04afc17,
+        0x27f00213,
+        0x0012d00d,
+        0x020721f5,
+        0x048f21f5,
+        0x000000f8,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc
new file mode 100644
index 00000000000..a1a599124cf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc
@@ -0,0 +1,808 @@
+/* fuc microcode for nvc0 PGRAPH/HUB
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+/* To build:
+ *    m4 nvc0_grhub.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grhub.fuc.h
+ */
+
+.section nvc0_grhub_data
+include(`nvc0_graph.fuc')
+gpc_count:              .b32 0
+rop_count:              .b32 0
+cmd_queue:              queue_init
+hub_mmio_list_head:     .b32 0
+hub_mmio_list_tail:     .b32 0
+
+ctx_current:            .b32 0
+
+chipsets:
+.b8  0xc0 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xc1 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc1_hub_mmio_tail
+.b8  0xc3 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xc4 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xc8 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xce 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0 0 0 0
+
+nvc0_hub_mmio_head:
+mmctx_data(0x17e91c, 2)
+mmctx_data(0x400204, 2)
+mmctx_data(0x404004, 11)
+mmctx_data(0x404044, 1)
+mmctx_data(0x404094, 14)
+mmctx_data(0x4040d0, 7)
+mmctx_data(0x4040f8, 1)
+mmctx_data(0x404130, 3)
+mmctx_data(0x404150, 3)
+mmctx_data(0x404164, 2)
+mmctx_data(0x404174, 3)
+mmctx_data(0x404200, 8)
+mmctx_data(0x404404, 14)
+mmctx_data(0x404460, 4)
+mmctx_data(0x404480, 1)
+mmctx_data(0x404498, 1)
+mmctx_data(0x404604, 4)
+mmctx_data(0x404618, 32)
+mmctx_data(0x404698, 21)
+mmctx_data(0x4046f0, 2)
+mmctx_data(0x404700, 22)
+mmctx_data(0x405800, 1)
+mmctx_data(0x405830, 3)
+mmctx_data(0x405854, 1)
+mmctx_data(0x405870, 4)
+mmctx_data(0x405a00, 2)
+mmctx_data(0x405a18, 1)
+mmctx_data(0x406020, 1)
+mmctx_data(0x406028, 4)
+mmctx_data(0x4064a8, 2)
+mmctx_data(0x4064b4, 2)
+mmctx_data(0x407804, 1)
+mmctx_data(0x40780c, 6)
+mmctx_data(0x4078bc, 1)
+mmctx_data(0x408000, 7)
+mmctx_data(0x408064, 1)
+mmctx_data(0x408800, 3)
+mmctx_data(0x408900, 4)
+mmctx_data(0x408980, 1)
+nvc0_hub_mmio_tail:
+mmctx_data(0x4064c0, 2)
+nvc1_hub_mmio_tail:
+
+.align 256
+chan_data:
+chan_mmio_count:        .b32 0
+chan_mmio_address:      .b32 0
+
+.align 256
+xfer_data:              .b32 0
+
+.section nvc0_grhub_code
+bra init
+define(`include_code')
+include(`nvc0_graph.fuc')
+
+// reports an exception to the host
+//
+// In: $r15 error code (see nvc0_graph.fuc)
+//
+error:
+        push $r14
+        mov $r14 0x814
+        shl b32 $r14 6
+        iowr I[$r14 + 0x000] $r15       // CC_SCRATCH[5] = error code
+        mov $r14 0xc1c
+        shl b32 $r14 6
+        mov $r15 1
+        iowr I[$r14 + 0x000] $r15       // INTR_UP_SET
+        pop $r14
+        ret
+
+// HUB fuc initialisation, executed by triggering ucode start, will
+// fall through to main loop after completion.
+//
+// Input:
+//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
+//
+// Output:
+//   CC_SCRATCH[0]:
+//           31:31: set to signal completion
+//   CC_SCRATCH[1]:
+//            31:0: total PGRAPH context size
+//
+init:
+        clear b32 $r0
+        mov $sp $r0
+        mov $xdbase $r0
+
+        // enable fifo access
+        mov $r1 0x1200
+        mov $r2 2
+        iowr I[$r1 + 0x000] $r2 // FIFO_ENABLE
+
+        // setup i0 handler, and route all interrupts to it
+        mov $r1 ih
+        mov $iv0 $r1
+        mov $r1 0x400
+        iowr I[$r1 + 0x300] $r0 // INTR_DISPATCH
+
+        // route HUB_CHANNEL_SWITCH to fuc interrupt 8
+        mov $r3 0x404
+        shl b32 $r3 6
+        mov $r2 0x2003          // { HUB_CHANNEL_SWITCH, ZERO } -> intr 8
+        iowr I[$r3 + 0x000] $r2
+
+        // not sure what these are, route them because NVIDIA does, and
+        // the IRQ handler will signal the host if we ever get one.. we
+        // may find out if/why we need to handle these if so..
+        //
+        mov $r2 0x2004
+        iowr I[$r3 + 0x004] $r2 // { 0x04, ZERO } -> intr 9
+        mov $r2 0x200b
+        iowr I[$r3 + 0x008] $r2 // { 0x0b, ZERO } -> intr 10
+        mov $r2 0x200c
+        iowr I[$r3 + 0x01c] $r2 // { 0x0c, ZERO } -> intr 15
+
+        // enable all INTR_UP interrupts
+        mov $r2 0xc24
+        shl b32 $r2 6
+        not b32 $r3 $r0
+        iowr I[$r2] $r3
+
+        // enable fifo, ctxsw, 9, 10, 15 interrupts
+        mov $r2 -0x78fc         // 0x8704
+        sethi $r2 0
+        iowr I[$r1 + 0x000] $r2 // INTR_EN_SET
+
+        // fifo level triggered, rest edge
+        sub b32 $r1 0x100
+        mov $r2 4
+        iowr I[$r1] $r2
+
+        // enable interrupts
+        bset $flags ie0
+
+        // fetch enabled GPC/ROP counts
+        mov $r14 -0x69fc        // 0x409604
+        sethi $r14 0x400000
+        call nv_rd32
+        extr $r1 $r15 16:20
+        st b32 D[$r0 + rop_count] $r1
+        and $r15 0x1f
+        st b32 D[$r0 + gpc_count] $r15
+
+        // set BAR_REQMASK to GPC mask
+        mov $r1 1
+        shl b32 $r1 $r15
+        sub b32 $r1 1
+        mov $r2 0x40c
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r1
+        iowr I[$r2 + 0x100] $r1
+
+        // find context data for this chipset
+        mov $r2 0x800
+        shl b32 $r2 6
+        iord $r2 I[$r2 + 0x000]         // CC_SCRATCH[0]
+        mov $r15 chipsets - 8
+        init_find_chipset:
+                add b32 $r15 8
+                ld b32 $r3 D[$r15 + 0x00]
+                cmpu b32 $r3 $r2
+                bra e init_context
+                cmpu b32 $r3 0
+                bra ne init_find_chipset
+                // unknown chipset
+                ret
+
+        // context size calculation, reserve first 256 bytes for use by fuc
+        init_context:
+        mov $r1 256
+
+        // calculate size of mmio context data
+        ld b16 $r14 D[$r15 + 4]
+        ld b16 $r15 D[$r15 + 6]
+        sethi $r14 0
+        st b32 D[$r0 + hub_mmio_list_head] $r14
+        st b32 D[$r0 + hub_mmio_list_tail] $r15
+        call mmctx_size
+
+        // set mmctx base addresses now so we don't have to do it later,
+        // they don't (currently) ever change
+        mov $r3 0x700
+        shl b32 $r3 6
+        shr b32 $r4 $r1 8
+        iowr I[$r3 + 0x000] $r4         // MMCTX_SAVE_SWBASE
+        iowr I[$r3 + 0x100] $r4         // MMCTX_LOAD_SWBASE
+        add b32 $r3 0x1300
+        add b32 $r1 $r15
+        shr b32 $r15 2
+        iowr I[$r3 + 0x000] $r15        // MMCTX_LOAD_COUNT, wtf for?!?
+
+        // strands, base offset needs to be aligned to 256 bytes
+        shr b32 $r1 8
+        add b32 $r1 1
+        shl b32 $r1 8
+        mov b32 $r15 $r1
+        call strand_ctx_init
+        add b32 $r1 $r15
+
+        // initialise each GPC in sequence by passing in the offset of its
+        // context data in GPCn_CC_SCRATCH[1], and starting its FUC (which
+        // has previously been uploaded by the host) running.
+        //
+        // the GPC fuc init sequence will set GPCn_CC_SCRATCH[0] bit 31
+        // when it has completed, and return the size of its context data
+        // in GPCn_CC_SCRATCH[1]
+        //
+        ld b32 $r3 D[$r0 + gpc_count]
+        mov $r4 0x2000
+        sethi $r4 0x500000
+        init_gpc:
+                // setup, and start GPC ucode running
+                add b32 $r14 $r4 0x804
+                mov b32 $r15 $r1
+                call nv_wr32                    // CC_SCRATCH[1] = ctx offset
+                add b32 $r14 $r4 0x800
+                mov b32 $r15 $r2
+                call nv_wr32                    // CC_SCRATCH[0] = chipset
+                add b32 $r14 $r4 0x10c
+                clear b32 $r15
+                call nv_wr32
+                add b32 $r14 $r4 0x104
+                call nv_wr32                    // ENTRY
+                add b32 $r14 $r4 0x100
+                mov $r15 2                      // CTRL_START_TRIGGER
+                call nv_wr32                    // CTRL
+
+                // wait for it to complete, and adjust context size
+                add b32 $r14 $r4 0x800
+                init_gpc_wait:
+                        call nv_rd32
+                        xbit $r15 $r15 31
+                        bra e init_gpc_wait
+                add b32 $r14 $r4 0x804
+                call nv_rd32
+                add b32 $r1 $r15
+
+                // next!
+                add b32 $r4 0x8000
+                sub b32 $r3 1
+                bra ne init_gpc
+
+        // save context size, and tell host we're ready
+        mov $r2 0x800
+        shl b32 $r2 6
+        iowr I[$r2 + 0x100] $r1         // CC_SCRATCH[1]  = context size
+        add b32 $r2 0x800
+        clear b32 $r1
+        bset $r1 31
+        iowr I[$r2 + 0x000] $r1         // CC_SCRATCH[0] |= 0x80000000
+
+// Main program loop, very simple, sleeps until woken up by the interrupt
+// handler, pulls a command from the queue and executes its handler
+//
+main:
+        // sleep until we have something to do
+        bset $flags $p0
+        sleep $p0
+        mov $r13 cmd_queue
+        call queue_get
+        bra $p1 main
+
+        // context switch, requested by GPU?
+        cmpu b32 $r14 0x4001
+        bra ne main_not_ctx_switch
+                trace_set(T_AUTO)
+                mov $r1 0xb00
+                shl b32 $r1 6
+                iord $r2 I[$r1 + 0x100]         // CHAN_NEXT
+                iord $r1 I[$r1 + 0x000]         // CHAN_CUR
+
+                xbit $r3 $r1 31
+                bra e chsw_no_prev
+                        xbit $r3 $r2 31
+                        bra e chsw_prev_no_next
+                                push $r2
+                                mov b32 $r2 $r1
+                                trace_set(T_SAVE)
+                                bclr $flags $p1
+                                bset $flags $p2
+                                call ctx_xfer
+                                trace_clr(T_SAVE);
+                                pop $r2
+                                trace_set(T_LOAD);
+                                bset $flags $p1
+                                call ctx_xfer
+                                trace_clr(T_LOAD);
+                                bra chsw_done
+                        chsw_prev_no_next:
+                                push $r2
+                                mov b32 $r2 $r1
+                                bclr $flags $p1
+                                bclr $flags $p2
+                                call ctx_xfer
+                                pop $r2
+                                mov $r1 0xb00
+                                shl b32 $r1 6
+                                iowr I[$r1] $r2
+                                bra chsw_done
+                chsw_no_prev:
+                        xbit $r3 $r2 31
+                        bra e chsw_done
+                                bset $flags $p1
+                                bclr $flags $p2
+                                call ctx_xfer
+
+                // ack the context switch request
+                chsw_done:
+                mov $r1 0xb0c
+                shl b32 $r1 6
+                mov $r2 1
+                iowr I[$r1 + 0x000] $r2         // 0x409b0c
+                trace_clr(T_AUTO)
+                bra main
+
+        // request to set current channel? (*not* a context switch)
+        main_not_ctx_switch:
+        cmpu b32 $r14 0x0001
+        bra ne main_not_ctx_chan
+                mov b32 $r2 $r15
+                call ctx_chan
+                bra main_done
+
+        // request to store current channel context?
+        main_not_ctx_chan:
+        cmpu b32 $r14 0x0002
+        bra ne main_not_ctx_save
+                trace_set(T_SAVE)
+                bclr $flags $p1
+                bclr $flags $p2
+                call ctx_xfer
+                trace_clr(T_SAVE)
+                bra main_done
+
+        main_not_ctx_save:
+                shl b32 $r15 $r14 16
+                or $r15 E_BAD_COMMAND
+                call error
+                bra main
+
+        main_done:
+        mov $r1 0x820
+        shl b32 $r1 6
+        clear b32 $r2
+        bset $r2 31
+        iowr I[$r1 + 0x000] $r2         // CC_SCRATCH[0] |= 0x80000000
+        bra main
+
+// interrupt handler
+ih:
+        push $r8
+        mov $r8 $flags
+        push $r8
+        push $r9
+        push $r10
+        push $r11
+        push $r13
+        push $r14
+        push $r15
+
+        // incoming fifo command?
+        iord $r10 I[$r0 + 0x200]        // INTR
+        and $r11 $r10 0x00000004
+        bra e ih_no_fifo
+                // queue incoming fifo command for later processing
+                mov $r11 0x1900
+                mov $r13 cmd_queue
+                iord $r14 I[$r11 + 0x100]       // FIFO_CMD
+                iord $r15 I[$r11 + 0x000]       // FIFO_DATA
+                call queue_put
+                add b32 $r11 0x400
+                mov $r14 1
+                iowr I[$r11 + 0x000] $r14       // FIFO_ACK
+
+        // context switch request?
+        ih_no_fifo:
+        and $r11 $r10 0x00000100
+        bra e ih_no_ctxsw
+                // enqueue a context switch for later processing
+                mov $r13 cmd_queue
+                mov $r14 0x4001
+                call queue_put
+
+        // anything we didn't handle, bring it to the host's attention
+        ih_no_ctxsw:
+        mov $r11 0x104
+        not b32 $r11
+        and $r11 $r10 $r11
+        bra e ih_no_other
+                mov $r10 0xc1c
+                shl b32 $r10 6
+                iowr I[$r10] $r11       // INTR_UP_SET
+
+        // ack, and wake up main()
+        ih_no_other:
+        iowr I[$r0 + 0x100] $r10        // INTR_ACK
+
+        pop $r15
+        pop $r14
+        pop $r13
+        pop $r11
+        pop $r10
+        pop $r9
+        pop $r8
+        mov $flags $r8
+        pop $r8
+        bclr $flags $p0
+        iret
+
+// Not real sure, but, MEM_CMD 7 will hang forever if this isn't done
+ctx_4160s:
+        mov $r14 0x4160
+        sethi $r14 0x400000
+        mov $r15 1
+        call nv_wr32
+        ctx_4160s_wait:
+                call nv_rd32
+                xbit $r15 $r15 4
+                bra e ctx_4160s_wait
+        ret
+
+// Without clearing again at end of xfer, some things cause PGRAPH
+// to hang with STATUS=0x00000007 until it's cleared.. fbcon can
+// still function with it set however...
+ctx_4160c:
+        mov $r14 0x4160
+        sethi $r14 0x400000
+        clear b32 $r15
+        call nv_wr32
+        ret
+
+// Again, not real sure
+//
+// In: $r15 value to set 0x404170 to
+//
+ctx_4170s:
+        mov $r14 0x4170
+        sethi $r14 0x400000
+        or $r15 0x10
+        call nv_wr32
+        ret
+
+// Waits for a ctx_4170s() call to complete
+//
+ctx_4170w:
+        mov $r14 0x4170
+        sethi $r14 0x400000
+        call nv_rd32
+        and $r15 0x10
+        bra ne ctx_4170w
+        ret
+
+// Disables various things, waits a bit, and re-enables them..
+//
+// Not sure how exactly this helps, perhaps "ENABLE" is not such a
+// good description for the bits we turn off?  Anyways, without this,
+// funny things happen.
+//
+ctx_redswitch:
+        mov $r14 0x614
+        shl b32 $r14 6
+        mov $r15 0x270
+        iowr I[$r14] $r15       // HUB_RED_SWITCH = ENABLE_GPC, POWER_ALL
+        mov $r15 8
+        ctx_redswitch_delay:
+                sub b32 $r15 1
+                bra ne ctx_redswitch_delay
+        mov $r15 0x770
+        iowr I[$r14] $r15       // HUB_RED_SWITCH = ENABLE_ALL, POWER_ALL
+        ret
+
+// Not a clue what this is for, except that unless the value is 0x10, the
+// strand context is saved (and presumably restored) incorrectly..
+//
+// In: $r15 value to set to (0x00/0x10 are used)
+//
+ctx_86c:
+        mov $r14 0x86c
+        shl b32 $r14 6
+        iowr I[$r14] $r15       // HUB(0x86c) = val
+        mov $r14 -0x75ec
+        sethi $r14 0x400000
+        call nv_wr32            // ROP(0xa14) = val
+        mov $r14 -0x5794
+        sethi $r14 0x410000
+        call nv_wr32            // GPC(0x86c) = val
+        ret
+
+// ctx_load - load's a channel's ctxctl data, and selects its vm
+//
+// In: $r2 channel address
+//
+ctx_load:
+        trace_set(T_CHAN)
+
+        // switch to channel, somewhat magic in parts..
+        mov $r10 12             // DONE_UNK12
+        call wait_donez
+        mov $r1 0xa24
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r0 // 0x409a24
+        mov $r3 0xb00
+        shl b32 $r3 6
+        iowr I[$r3 + 0x100] $r2 // CHAN_NEXT
+        mov $r1 0xa0c
+        shl b32 $r1 6
+        mov $r4 7
+        iowr I[$r1 + 0x000] $r2 // MEM_CHAN
+        iowr I[$r1 + 0x100] $r4 // MEM_CMD
+        ctx_chan_wait_0:
+                iord $r4 I[$r1 + 0x100]
+                and $r4 0x1f
+                bra ne ctx_chan_wait_0
+        iowr I[$r3 + 0x000] $r2 // CHAN_CUR
+
+        // load channel header, fetch PGRAPH context pointer
+        mov $xtargets $r0
+        bclr $r2 31
+        shl b32 $r2 4
+        add b32 $r2 2
+
+        trace_set(T_LCHAN)
+        mov $r1 0xa04
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r2         // MEM_BASE
+        mov $r1 0xa20
+        shl b32 $r1 6
+        mov $r2 0x0002
+        sethi $r2 0x80000000
+        iowr I[$r1 + 0x000] $r2         // MEM_TARGET = vram
+        mov $r1 0x10                    // chan + 0x0210
+        mov $r2 xfer_data
+        sethi $r2 0x00020000            // 16 bytes
+        xdld $r1 $r2
+        xdwait
+        trace_clr(T_LCHAN)
+
+        // update current context
+        ld b32 $r1 D[$r0 + xfer_data + 4]
+        shl b32 $r1 24
+        ld b32 $r2 D[$r0 + xfer_data + 0]
+        shr b32 $r2 8
+        or $r1 $r2
+        st b32 D[$r0 + ctx_current] $r1
+
+        // set transfer base to start of context, and fetch context header
+        trace_set(T_LCTXH)
+        mov $r2 0xa04
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r1         // MEM_BASE
+        mov $r2 1
+        mov $r1 0xa20
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r2         // MEM_TARGET = vm
+        mov $r1 chan_data
+        sethi $r1 0x00060000            // 256 bytes
+        xdld $r0 $r1
+        xdwait
+        trace_clr(T_LCTXH)
+
+        trace_clr(T_CHAN)
+        ret
+
+// ctx_chan - handler for HUB_SET_CHAN command, will set a channel as
+//            the active channel for ctxctl, but not actually transfer
+//            any context data.  intended for use only during initial
+//            context construction.
+//
+// In: $r2 channel address
+//
+ctx_chan:
+        call ctx_4160s
+        call ctx_load
+        mov $r10 12                     // DONE_UNK12
+        call wait_donez
+        mov $r1 0xa10
+        shl b32 $r1 6
+        mov $r2 5
+        iowr I[$r1 + 0x000] $r2         // MEM_CMD = 5 (???)
+        ctx_chan_wait:
+                iord $r2 I[$r1 + 0x000]
+                or $r2 $r2
+                bra ne ctx_chan_wait
+        call ctx_4160c
+        ret
+
+// Execute per-context state overrides list
+//
+// Only executed on the first load of a channel.  Might want to look into
+// removing this and having the host directly modify the channel's context
+// to change this state...  The nouveau DRM already builds this list as
+// it's definitely needed for NVIDIA's, so we may as well use it for now
+//
+// Input: $r1 mmio list length
+//
+ctx_mmio_exec:
+        // set transfer base to be the mmio list
+        ld b32 $r3 D[$r0 + chan_mmio_address]
+        mov $r2 0xa04
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r3         // MEM_BASE
+
+        clear b32 $r3
+        ctx_mmio_loop:
+                // fetch next 256 bytes of mmio list if necessary
+                and $r4 $r3 0xff
+                bra ne ctx_mmio_pull
+                        mov $r5 xfer_data
+                        sethi $r5 0x00060000    // 256 bytes
+                        xdld $r3 $r5
+                        xdwait
+
+                // execute a single list entry
+                ctx_mmio_pull:
+                ld b32 $r14 D[$r4 + xfer_data + 0x00]
+                ld b32 $r15 D[$r4 + xfer_data + 0x04]
+                call nv_wr32
+
+                // next!
+                add b32 $r3 8
+                sub b32 $r1 1
+                bra ne ctx_mmio_loop
+
+        // set transfer base back to the current context
+        ctx_mmio_done:
+        ld b32 $r3 D[$r0 + ctx_current]
+        iowr I[$r2 + 0x000] $r3         // MEM_BASE
+
+        // disable the mmio list now, we don't need/want to execute it again
+        st b32 D[$r0 + chan_mmio_count] $r0
+        mov $r1 chan_data
+        sethi $r1 0x00060000            // 256 bytes
+        xdst $r0 $r1
+        xdwait
+        ret
+
+// Transfer HUB context data between GPU and storage area
+//
+// In: $r2 channel address
+//     $p1 clear on save, set on load
+//     $p2 set if opposite direction done/will be done, so:
+//              on save it means: "a load will follow this save"
+//              on load it means: "a save preceeded this load"
+//
+ctx_xfer:
+        bra not $p1 ctx_xfer_pre
+        bra $p2 ctx_xfer_pre_load
+        ctx_xfer_pre:
+                mov $r15 0x10
+                call ctx_86c
+                call ctx_4160s
+                bra not $p1 ctx_xfer_exec
+
+        ctx_xfer_pre_load:
+                mov $r15 2
+                call ctx_4170s
+                call ctx_4170w
+                call ctx_redswitch
+                clear b32 $r15
+                call ctx_4170s
+                call ctx_load
+
+        // fetch context pointer, and initiate xfer on all GPCs
+        ctx_xfer_exec:
+        ld b32 $r1 D[$r0 + ctx_current]
+        mov $r2 0x414
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r0 // BAR_STATUS = reset
+        mov $r14 -0x5b00
+        sethi $r14 0x410000
+        mov b32 $r15 $r1
+        call nv_wr32            // GPC_BCAST_WRCMD_DATA = ctx pointer
+        add b32 $r14 4
+        xbit $r15 $flags $p1
+        xbit $r2 $flags $p2
+        shl b32 $r2 1
+        or $r15 $r2
+        call nv_wr32            // GPC_BCAST_WRCMD_CMD = GPC_XFER(type)
+
+        // strands
+        mov $r1 0x4afc
+        sethi $r1 0x20000
+        mov $r2 0xc
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x0c
+        call strand_wait
+        mov $r2 0x47fc
+        sethi $r2 0x20000
+        iowr I[$r2] $r0         // STRAND_FIRST_GENE(0x3f) = 0x00
+        xbit $r2 $flags $p1
+        add b32 $r2 3
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)
+
+        // mmio context
+        xbit $r10 $flags $p1    // direction
+        or $r10 6               // first, last
+        mov $r11 0              // base = 0
+        ld b32 $r12 D[$r0 + hub_mmio_list_head]
+        ld b32 $r13 D[$r0 + hub_mmio_list_tail]
+        mov $r14 0              // not multi
+        call mmctx_xfer
+
+        // wait for GPCs to all complete
+        mov $r10 8              // DONE_BAR
+        call wait_doneo
+
+        // wait for strand xfer to complete
+        call strand_wait
+
+        // post-op
+        bra $p1 ctx_xfer_post
+                mov $r10 12             // DONE_UNK12
+                call wait_donez
+                mov $r1 0xa10
+                shl b32 $r1 6
+                mov $r2 5
+                iowr I[$r1] $r2         // MEM_CMD
+                ctx_xfer_post_save_wait:
+                        iord $r2 I[$r1]
+                        or $r2 $r2
+                        bra ne ctx_xfer_post_save_wait
+
+        bra $p2 ctx_xfer_done
+        ctx_xfer_post:
+                mov $r15 2
+                call ctx_4170s
+                clear b32 $r15
+                call ctx_86c
+                call strand_post
+                call ctx_4170w
+                clear b32 $r15
+                call ctx_4170s
+
+                bra not $p1 ctx_xfer_no_post_mmio
+                ld b32 $r1 D[$r0 + chan_mmio_count]
+                or $r1 $r1
+                bra e ctx_xfer_no_post_mmio
+                        call ctx_mmio_exec
+
+                ctx_xfer_no_post_mmio:
+                call ctx_4160c
+
+        ctx_xfer_done:
+        ret
+
+.align 256
diff --git a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h
new file mode 100644
index 00000000000..b3b541b6d04
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h
@@ -0,0 +1,838 @@
+uint32_t nvc0_grhub_data[] = {
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x000000c0,
+        0x012c0090,
+        0x000000c1,
+        0x01300090,
+        0x000000c3,
+        0x012c0090,
+        0x000000c4,
+        0x012c0090,
+        0x000000c8,
+        0x012c0090,
+        0x000000ce,
+        0x012c0090,
+        0x00000000,
+        0x0417e91c,
+        0x04400204,
+        0x28404004,
+        0x00404044,
+        0x34404094,
+        0x184040d0,
+        0x004040f8,
+        0x08404130,
+        0x08404150,
+        0x04404164,
+        0x08404174,
+        0x1c404200,
+        0x34404404,
+        0x0c404460,
+        0x00404480,
+        0x00404498,
+        0x0c404604,
+        0x7c404618,
+        0x50404698,
+        0x044046f0,
+        0x54404700,
+        0x00405800,
+        0x08405830,
+        0x00405854,
+        0x0c405870,
+        0x04405a00,
+        0x00405a18,
+        0x00406020,
+        0x0c406028,
+        0x044064a8,
+        0x044064b4,
+        0x00407804,
+        0x1440780c,
+        0x004078bc,
+        0x18408000,
+        0x00408064,
+        0x08408800,
+        0x0c408900,
+        0x00408980,
+        0x044064c0,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+
+uint32_t nvc0_grhub_code[] = {
+        0x03090ef5,
+        0x9800d898,
+        0x86f001d9,
+        0x0489b808,
+        0xf00c1bf4,
+        0x21f502f7,
+        0x00f802ec,
+        0xb60798c4,
+        0x8dbb0384,
+        0x0880b600,
+        0x80008e80,
+        0x90b6018f,
+        0x0f94f001,
+        0xf801d980,
+        0x0131f400,
+        0x9800d898,
+        0x89b801d9,
+        0x210bf404,
+        0xb60789c4,
+        0x9dbb0394,
+        0x0890b600,
+        0x98009e98,
+        0x80b6019f,
+        0x0f84f001,
+        0xf400d880,
+        0x00f80132,
+        0x0728b7f1,
+        0xb906b4b6,
+        0xc9f002ec,
+        0x00bcd01f,
+        0xc800bccf,
+        0x1bf41fcc,
+        0x06a7f0fa,
+        0x010321f5,
+        0xf840bfcf,
+        0x28b7f100,
+        0x06b4b607,
+        0xb980bfd0,
+        0xc9f002ec,
+        0x1ec9f01f,
+        0xcf00bcd0,
+        0xccc800bc,
+        0xfa1bf41f,
+        0x87f100f8,
+        0x84b60430,
+        0x1ff9f006,
+        0xf8008fd0,
+        0x3087f100,
+        0x0684b604,
+        0xf80080d0,
+        0x3c87f100,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d000,
+        0x081887f1,
+        0xd00684b6,
+        0x87f1008a,
+        0x84b60400,
+        0x0088cf06,
+        0xf4888aff,
+        0x87f1f31b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00099,
+        0xf100f800,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00099f0,
+        0x87f10089,
+        0x84b60818,
+        0x008ad006,
+        0x040087f1,
+        0xcf0684b6,
+        0x8aff0088,
+        0xf30bf488,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0099f094,
+        0xf80089d0,
+        0x9894bd00,
+        0x85b600e8,
+        0x0180b61a,
+        0xbb0284b6,
+        0xe0b60098,
+        0x04efb804,
+        0xb9eb1bf4,
+        0x00f8029f,
+        0x083c87f1,
+        0xbd0684b6,
+        0x0199f094,
+        0xf10089d0,
+        0xb6071087,
+        0x94bd0684,
+        0xf405bbfd,
+        0x8bd0090b,
+        0x0099f000,
+        0xf405eefd,
+        0x8ed00c0b,
+        0xc08fd080,
+        0xb70199f0,
+        0xc8010080,
+        0xb4b600ab,
+        0x0cb9f010,
+        0xb601aec8,
+        0xbefd11e4,
+        0x008bd005,
+        0xf0008ecf,
+        0x0bf41fe4,
+        0x00ce98fa,
+        0xd005e9fd,
+        0xc0b6c08e,
+        0x04cdb804,
+        0xc8e81bf4,
+        0x1bf402ab,
+        0x008bcf18,
+        0xb01fb4f0,
+        0x1bf410b4,
+        0x02a7f0f7,
+        0xf4c921f4,
+        0xabc81b0e,
+        0x10b4b600,
+        0xf00cb9f0,
+        0x8bd012b9,
+        0x008bcf00,
+        0xf412bbc8,
+        0x87f1fa1b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00199,
+        0xf900f800,
+        0x02a7f0a0,
+        0xfcc921f4,
+        0xf100f8a0,
+        0xf04afc87,
+        0x97f00283,
+        0x0089d00c,
+        0x020721f5,
+        0x87f100f8,
+        0x83f04afc,
+        0x0d97f002,
+        0xf50089d0,
+        0xf8020721,
+        0xfca7f100,
+        0x02a3f04f,
+        0x0500aba2,
+        0xd00fc7f0,
+        0xc7f000ac,
+        0x00bcd00b,
+        0x020721f5,
+        0xf000aed0,
+        0xbcd00ac7,
+        0x0721f500,
+        0xf100f802,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x21f50089,
+        0xe7f00213,
+        0x3921f503,
+        0xfca7f102,
+        0x02a3f046,
+        0x0400aba0,
+        0xf040a0d0,
+        0xbcd001c7,
+        0x0721f500,
+        0x010c9202,
+        0xf000acd0,
+        0xbcd002c7,
+        0x0721f500,
+        0x2621f502,
+        0x8087f102,
+        0x0684b608,
+        0xb70089cf,
+        0x95220080,
+        0x8ed008fe,
+        0x408ed000,
+        0xb6808acf,
+        0xa0b606a5,
+        0x00eabb01,
+        0xb60480b6,
+        0x1bf40192,
+        0x08e4b6e8,
+        0xf1f2efbc,
+        0xb6085c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x00f80089,
+        0xe7f1e0f9,
+        0xe4b60814,
+        0x00efd006,
+        0x0c1ce7f1,
+        0xf006e4b6,
+        0xefd001f7,
+        0xf8e0fc00,
+        0xfe04bd00,
+        0x07fe0004,
+        0x0017f100,
+        0x0227f012,
+        0xf10012d0,
+        0xfe05b917,
+        0x17f10010,
+        0x10d00400,
+        0x0437f1c0,
+        0x0634b604,
+        0x200327f1,
+        0xf10032d0,
+        0xd0200427,
+        0x27f10132,
+        0x32d0200b,
+        0x0c27f102,
+        0x0732d020,
+        0x0c2427f1,
+        0xb90624b6,
+        0x23d00003,
+        0x0427f100,
+        0x0023f087,
+        0xb70012d0,
+        0xf0010012,
+        0x12d00427,
+        0x1031f400,
+        0x9604e7f1,
+        0xf440e3f0,
+        0xf1c76821,
+        0x01018090,
+        0x801ff4f0,
+        0x17f0000f,
+        0x041fbb01,
+        0xf10112b6,
+        0xb6040c27,
+        0x21d00624,
+        0x4021d000,
+        0x080027f1,
+        0xcf0624b6,
+        0xf7f00022,
+        0x08f0b654,
+        0xb800f398,
+        0x0bf40432,
+        0x0034b00b,
+        0xf8f11bf4,
+        0x0017f100,
+        0x02fe5801,
+        0xf003ff58,
+        0x0e8000e3,
+        0x150f8014,
+        0x013d21f5,
+        0x070037f1,
+        0x950634b6,
+        0x34d00814,
+        0x4034d000,
+        0x130030b7,
+        0xb6001fbb,
+        0x3fd002f5,
+        0x0815b600,
+        0xb60110b6,
+        0x1fb90814,
+        0x6321f502,
+        0x001fbb02,
+        0xf1000398,
+        0xf0200047,
+        0x4ea05043,
+        0x1fb90804,
+        0x8d21f402,
+        0x08004ea0,
+        0xf4022fb9,
+        0x4ea08d21,
+        0xf4bd010c,
+        0xa08d21f4,
+        0xf401044e,
+        0x4ea08d21,
+        0xf7f00100,
+        0x8d21f402,
+        0x08004ea0,
+        0xc86821f4,
+        0x0bf41fff,
+        0x044ea0fa,
+        0x6821f408,
+        0xb7001fbb,
+        0xb6800040,
+        0x1bf40132,
+        0x0027f1b4,
+        0x0624b608,
+        0xb74021d0,
+        0xbd080020,
+        0x1f19f014,
+        0xf40021d0,
+        0x28f40031,
+        0x08d7f000,
+        0xf43921f4,
+        0xe4b1f401,
+        0x1bf54001,
+        0x87f100d1,
+        0x84b6083c,
+        0xf094bd06,
+        0x89d00499,
+        0x0017f100,
+        0x0614b60b,
+        0xcf4012cf,
+        0x13c80011,
+        0x7e0bf41f,
+        0xf41f23c8,
+        0x20f95a0b,
+        0xf10212b9,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00799f0,
+        0x32f40089,
+        0x0231f401,
+        0x082921f5,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0799f094,
+        0xfc0089d0,
+        0x3c87f120,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d006,
+        0xf50131f4,
+        0xf1082921,
+        0xb6085c87,
+        0x94bd0684,
+        0xd00699f0,
+        0x0ef40089,
+        0xb920f931,
+        0x32f40212,
+        0x0232f401,
+        0x082921f5,
+        0x17f120fc,
+        0x14b60b00,
+        0x0012d006,
+        0xc8130ef4,
+        0x0bf41f23,
+        0x0131f40d,
+        0xf50232f4,
+        0xf1082921,
+        0xb60b0c17,
+        0x27f00614,
+        0x0012d001,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0499f094,
+        0xf50089d0,
+        0xb0ff200e,
+        0x1bf401e4,
+        0x02f2b90d,
+        0x07b521f5,
+        0xb0420ef4,
+        0x1bf402e4,
+        0x3c87f12e,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d007,
+        0xf40132f4,
+        0x21f50232,
+        0x87f10829,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00799,
+        0x110ef400,
+        0xf010ef94,
+        0x21f501f5,
+        0x0ef502ec,
+        0x17f1fed1,
+        0x14b60820,
+        0xf024bd06,
+        0x12d01f29,
+        0xbe0ef500,
+        0xfe80f9fe,
+        0x80f90188,
+        0xa0f990f9,
+        0xd0f9b0f9,
+        0xf0f9e0f9,
+        0xc4800acf,
+        0x0bf404ab,
+        0x00b7f11d,
+        0x08d7f019,
+        0xcf40becf,
+        0x21f400bf,
+        0x00b0b704,
+        0x01e7f004,
+        0xe400bed0,
+        0xf40100ab,
+        0xd7f00d0b,
+        0x01e7f108,
+        0x0421f440,
+        0x0104b7f1,
+        0xabffb0bd,
+        0x0d0bf4b4,
+        0x0c1ca7f1,
+        0xd006a4b6,
+        0x0ad000ab,
+        0xfcf0fc40,
+        0xfcd0fce0,
+        0xfca0fcb0,
+        0xfe80fc90,
+        0x80fc0088,
+        0xf80032f4,
+        0x60e7f101,
+        0x40e3f041,
+        0xf401f7f0,
+        0x21f48d21,
+        0x04ffc868,
+        0xf8fa0bf4,
+        0x60e7f100,
+        0x40e3f041,
+        0x21f4f4bd,
+        0xf100f88d,
+        0xf04170e7,
+        0xf5f040e3,
+        0x8d21f410,
+        0xe7f100f8,
+        0xe3f04170,
+        0x6821f440,
+        0xf410f4f0,
+        0x00f8f31b,
+        0x0614e7f1,
+        0xf106e4b6,
+        0xd00270f7,
+        0xf7f000ef,
+        0x01f2b608,
+        0xf1fd1bf4,
+        0xd00770f7,
+        0x00f800ef,
+        0x086ce7f1,
+        0xd006e4b6,
+        0xe7f100ef,
+        0xe3f08a14,
+        0x8d21f440,
+        0xa86ce7f1,
+        0xf441e3f0,
+        0x00f88d21,
+        0x083c87f1,
+        0xbd0684b6,
+        0x0599f094,
+        0xf00089d0,
+        0x21f40ca7,
+        0x2417f1c9,
+        0x0614b60a,
+        0xf10010d0,
+        0xb60b0037,
+        0x32d00634,
+        0x0c17f140,
+        0x0614b60a,
+        0xd00747f0,
+        0x14d00012,
+        0x4014cf40,
+        0xf41f44f0,
+        0x32d0fa1b,
+        0x000bfe00,
+        0xb61f2af0,
+        0x20b60424,
+        0x3c87f102,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d008,
+        0x0a0417f1,
+        0xd00614b6,
+        0x17f10012,
+        0x14b60a20,
+        0x0227f006,
+        0x800023f1,
+        0xf00012d0,
+        0x27f11017,
+        0x23f00300,
+        0x0512fa02,
+        0x87f103f8,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00899,
+        0xc1019800,
+        0x981814b6,
+        0x25b6c002,
+        0x0512fd08,
+        0xf1160180,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00999f0,
+        0x27f10089,
+        0x24b60a04,
+        0x0021d006,
+        0xf10127f0,
+        0xb60a2017,
+        0x12d00614,
+        0x0017f100,
+        0x0613f002,
+        0xf80501fa,
+        0x5c87f103,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d009,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0599f094,
+        0xf80089d0,
+        0x3121f500,
+        0xb821f506,
+        0x0ca7f006,
+        0xf1c921f4,
+        0xb60a1017,
+        0x27f00614,
+        0x0012d005,
+        0xfd0012cf,
+        0x1bf40522,
+        0x4921f5fa,
+        0x9800f806,
+        0x27f18103,
+        0x24b60a04,
+        0x0023d006,
+        0x34c434bd,
+        0x0f1bf4ff,
+        0x030057f1,
+        0xfa0653f0,
+        0x03f80535,
+        0x98c04e98,
+        0x21f4c14f,
+        0x0830b68d,
+        0xf40112b6,
+        0x0398df1b,
+        0x0023d016,
+        0xf1800080,
+        0xf0020017,
+        0x01fa0613,
+        0xf803f806,
+        0x0611f400,
+        0xf01102f4,
+        0x21f510f7,
+        0x21f50698,
+        0x11f40631,
+        0x02f7f01c,
+        0x065721f5,
+        0x066621f5,
+        0x067821f5,
+        0x21f5f4bd,
+        0x21f50657,
+        0x019806b8,
+        0x1427f116,
+        0x0624b604,
+        0xf10020d0,
+        0xf0a500e7,
+        0x1fb941e3,
+        0x8d21f402,
+        0xf004e0b6,
+        0x2cf001fc,
+        0x0124b602,
+        0xf405f2fd,
+        0x17f18d21,
+        0x13f04afc,
+        0x0c27f002,
+        0xf50012d0,
+        0xf1020721,
+        0xf047fc27,
+        0x20d00223,
+        0x012cf000,
+        0xd00320b6,
+        0xacf00012,
+        0x06a5f001,
+        0x9800b7f0,
+        0x0d98140c,
+        0x00e7f015,
+        0x015c21f5,
+        0xf508a7f0,
+        0xf5010321,
+        0xf4020721,
+        0xa7f02201,
+        0xc921f40c,
+        0x0a1017f1,
+        0xf00614b6,
+        0x12d00527,
+        0x0012cf00,
+        0xf40522fd,
+        0x02f4fa1b,
+        0x02f7f032,
+        0x065721f5,
+        0x21f5f4bd,
+        0x21f50698,
+        0x21f50226,
+        0xf4bd0666,
+        0x065721f5,
+        0x981011f4,
+        0x11fd8001,
+        0x070bf405,
+        0x07df21f5,
+        0x064921f5,
+        0x000000f8,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c
index 82357d2df1f..b701c439c92 100644
--- a/drivers/gpu/drm/nouveau/nvc0_instmem.c
+++ b/drivers/gpu/drm/nouveau/nvc0_instmem.c
@@ -32,7 +32,6 @@ struct nvc0_instmem_priv {
         struct nouveau_channel *bar1;
         struct nouveau_gpuobj  *bar3_pgd;
         struct nouveau_channel *bar3;
-        struct nouveau_gpuobj  *chan_pgd;
 };
 
 int
@@ -181,17 +180,11 @@ nvc0_instmem_init(struct drm_device *dev)
                 goto error;
 
         /* channel vm */
-        ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL, &vm);
+        ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL,
+                             &dev_priv->chan_vm);
         if (ret)
                 goto error;
 
-        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096, 0, &priv->chan_pgd);
-        if (ret)
-                goto error;
-
-        nouveau_vm_ref(vm, &dev_priv->chan_vm, priv->chan_pgd);
-        nouveau_vm_ref(NULL, &vm, NULL);
-
         nvc0_instmem_resume(dev);
         return 0;
 error:
@@ -211,8 +204,7 @@ nvc0_instmem_takedown(struct drm_device *dev)
         nv_wr32(dev, 0x1704, 0x00000000);
         nv_wr32(dev, 0x1714, 0x00000000);
 
-        nouveau_vm_ref(NULL, &dev_priv->chan_vm, priv->chan_pgd);
-        nouveau_gpuobj_ref(NULL, &priv->chan_pgd);
+        nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
 
         nvc0_channel_del(&priv->bar1);
         nouveau_vm_ref(NULL, &dev_priv->bar1_vm, priv->bar1_pgd);
diff --git a/drivers/gpu/drm/nouveau/nvc0_vm.c b/drivers/gpu/drm/nouveau/nvc0_vm.c
index a179e6c55af..9e352944a35 100644
--- a/drivers/gpu/drm/nouveau/nvc0_vm.c
+++ b/drivers/gpu/drm/nouveau/nvc0_vm.c
@@ -105,7 +105,11 @@ nvc0_vm_flush(struct nouveau_vm *vm)
         struct drm_device *dev = vm->dev;
         struct nouveau_vm_pgd *vpgd;
         unsigned long flags;
-        u32 engine = (dev_priv->chan_vm == vm) ? 1 : 5;
+        u32 engine;
+
+        engine = 1;
+        if (vm == dev_priv->bar1_vm || vm == dev_priv->bar3_vm)
+                engine |= 4;
 
         pinstmem->flush(vm->dev);
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_vram.c b/drivers/gpu/drm/nouveau/nvc0_vram.c
index 67c6ec6f34e..e45a24d84e9 100644
--- a/drivers/gpu/drm/nouveau/nvc0_vram.c
+++ b/drivers/gpu/drm/nouveau/nvc0_vram.c
@@ -61,9 +61,7 @@ nvc0_vram_new(struct drm_device *dev, u64 size, u32 align, u32 ncmin,
               u32 type, struct nouveau_mem **pmem)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
-        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
-        struct nouveau_mm *mm = man->priv;
+        struct nouveau_mm *mm = dev_priv->engine.vram.mm;
         struct nouveau_mm_node *r;
         struct nouveau_mem *mem;
         int ret;
@@ -105,9 +103,15 @@ int
 nvc0_vram_init(struct drm_device *dev)
 {
         struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+        const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+        u32 length;
 
         dev_priv->vram_size  = nv_rd32(dev, 0x10f20c) << 20;
         dev_priv->vram_size *= nv_rd32(dev, 0x121c74);
-        dev_priv->vram_rblock_size = 4096;
-        return 0;
+
+        length = (dev_priv->vram_size >> 12) - rsvd_head - rsvd_tail;
+
+        return nouveau_mm_init(&vram->mm, rsvd_head, length, 1);
 }
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index 3896ef81110..9f363e0c4b6 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -5,6 +5,7 @@
 ccflags-y := -Iinclude/drm
 
 hostprogs-y := mkregtable
+clean-files := rn50_reg_safe.h r100_reg_safe.h r200_reg_safe.h rv515_reg_safe.h r300_reg_safe.h r420_reg_safe.h rs600_reg_safe.h r600_reg_safe.h evergreen_reg_safe.h cayman_reg_safe.h
 
 quiet_cmd_mkregtable = MKREGTABLE $@
       cmd_mkregtable = $(obj)/mkregtable $< > $@
diff --git a/drivers/gpu/drm/radeon/atom.c b/drivers/gpu/drm/radeon/atom.c
index ebdb0fdb834..14cc88aaf3a 100644
--- a/drivers/gpu/drm/radeon/atom.c
+++ b/drivers/gpu/drm/radeon/atom.c
@@ -277,7 +277,12 @@ static uint32_t atom_get_src_int(atom_exec_context *ctx, uint8_t attr,
         case ATOM_ARG_FB:
                 idx = U8(*ptr);
                 (*ptr)++;
-                val = gctx->scratch[((gctx->fb_base + idx) / 4)];
+                if ((gctx->fb_base + (idx * 4)) > gctx->scratch_size_bytes) {
+                        DRM_ERROR("ATOM: fb read beyond scratch region: %d vs. %d\n",
+                                  gctx->fb_base + (idx * 4), gctx->scratch_size_bytes);
+                        val = 0;
+                } else
+                        val = gctx->scratch[(gctx->fb_base / 4) + idx];
                 if (print)
                         DEBUG("FB[0x%02X]", idx);
                 break;
@@ -531,7 +536,11 @@ static void atom_put_dst(atom_exec_context *ctx, int arg, uint8_t attr,
         case ATOM_ARG_FB:
                 idx = U8(*ptr);
                 (*ptr)++;
-                gctx->scratch[((gctx->fb_base + idx) / 4)] = val;
+                if ((gctx->fb_base + (idx * 4)) > gctx->scratch_size_bytes) {
+                        DRM_ERROR("ATOM: fb write beyond scratch region: %d vs. %d\n",
+                                  gctx->fb_base + (idx * 4), gctx->scratch_size_bytes);
+                } else
+                        gctx->scratch[(gctx->fb_base / 4) + idx] = val;
                 DEBUG("FB[0x%02X]", idx);
                 break;
         case ATOM_ARG_PLL:
@@ -1245,6 +1254,9 @@ struct atom_context *atom_parse(struct card_info *card, void *bios)
         char name[512];
         int i;
 
+        if (!ctx)
+                return NULL;
+
         ctx->card = card;
         ctx->bios = bios;
 
@@ -1367,11 +1379,13 @@ int atom_allocate_fb_scratch(struct atom_context *ctx)
 
                 usage_bytes = firmware_usage->asFirmwareVramReserveInfo[0].usFirmwareUseInKb * 1024;
         }
+        ctx->scratch_size_bytes = 0;
         if (usage_bytes == 0)
                 usage_bytes = 20 * 1024;
         /* allocate some scratch memory */
         ctx->scratch = kzalloc(usage_bytes, GFP_KERNEL);
         if (!ctx->scratch)
                 return -ENOMEM;
+        ctx->scratch_size_bytes = usage_bytes;
         return 0;
 }
diff --git a/drivers/gpu/drm/radeon/atom.h b/drivers/gpu/drm/radeon/atom.h
index a589a55b223..93cfe2086ba 100644
--- a/drivers/gpu/drm/radeon/atom.h
+++ b/drivers/gpu/drm/radeon/atom.h
@@ -137,6 +137,7 @@ struct atom_context {
         int cs_equal, cs_above;
         int io_mode;
         uint32_t *scratch;
+        int scratch_size_bytes;
 };
 
 extern int atom_debug;
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 9541995e4b2..9bb3d6f3b7b 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -466,7 +466,7 @@ static void atombios_crtc_program_ss(struct drm_crtc *crtc,
                         return;
                 }
                 args.v2.ucEnable = enable;
-                if ((ss->percentage == 0) || (ss->type & ATOM_EXTERNAL_SS_MASK))
+                if ((ss->percentage == 0) || (ss->type & ATOM_EXTERNAL_SS_MASK) || ASIC_IS_DCE41(rdev))
                         args.v2.ucEnable = ATOM_DISABLE;
         } else if (ASIC_IS_DCE3(rdev)) {
                 args.v1.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
@@ -558,7 +558,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 bpc = connector->display_info.bpc;
                         encoder_mode = atombios_get_encoder_mode(encoder);
                         if ((radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
-                            radeon_encoder_is_dp_bridge(encoder)) {
+                            (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)) {
                                 if (connector) {
                                         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
                                         struct radeon_connector_atom_dig *dig_connector =
@@ -638,44 +638,29 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
                                 if (ss_enabled && ss->percentage)
                                         args.v3.sInput.ucDispPllConfig |=
                                                 DISPPLL_CONFIG_SS_ENABLE;
-                                if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT) ||
-                                    radeon_encoder_is_dp_bridge(encoder)) {
+                                if (encoder_mode == ATOM_ENCODER_MODE_DP) {
+                                        args.v3.sInput.ucDispPllConfig |=
+                                                DISPPLL_CONFIG_COHERENT_MODE;
+                                        /* 16200 or 27000 */
+                                        args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10);
+                                } else if (radeon_encoder->devices & (ATOM_DEVICE_DFP_SUPPORT)) {
                                         struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
-                                        if (encoder_mode == ATOM_ENCODER_MODE_DP) {
+                                        if (encoder_mode == ATOM_ENCODER_MODE_HDMI)
+                                                /* deep color support */
+                                                args.v3.sInput.usPixelClock =
+                                                        cpu_to_le16((mode->clock * bpc / 8) / 10);
+                                        if (dig->coherent_mode)
                                                 args.v3.sInput.ucDispPllConfig |=
                                                         DISPPLL_CONFIG_COHERENT_MODE;
-                                                /* 16200 or 27000 */
-                                                args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10);
-                                        } else {
-                                                if (encoder_mode == ATOM_ENCODER_MODE_HDMI) {
-                                                        /* deep color support */
-                                                        args.v3.sInput.usPixelClock =
-                                                                cpu_to_le16((mode->clock * bpc / 8) / 10);
-                                                }
-                                                if (dig->coherent_mode)
-                                                        args.v3.sInput.ucDispPllConfig |=
-                                                                DISPPLL_CONFIG_COHERENT_MODE;
-                                                if (mode->clock > 165000)
-                                                        args.v3.sInput.ucDispPllConfig |=
-                                                                DISPPLL_CONFIG_DUAL_LINK;
-                                        }
-                                } else if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
-                                        if (encoder_mode == ATOM_ENCODER_MODE_DP) {
+                                        if (mode->clock > 165000)
                                                 args.v3.sInput.ucDispPllConfig |=
-                                                        DISPPLL_CONFIG_COHERENT_MODE;
-                                                /* 16200 or 27000 */
-                                                args.v3.sInput.usPixelClock = cpu_to_le16(dp_clock / 10);
-                                        } else if (encoder_mode != ATOM_ENCODER_MODE_LVDS) {
-                                                if (mode->clock > 165000)
-                                                        args.v3.sInput.ucDispPllConfig |=
-                                                                DISPPLL_CONFIG_DUAL_LINK;
-                                        }
+                                                        DISPPLL_CONFIG_DUAL_LINK;
                                 }
-                                if (radeon_encoder_is_dp_bridge(encoder)) {
-                                        struct drm_encoder *ext_encoder = radeon_atom_get_external_encoder(encoder);
-                                        struct radeon_encoder *ext_radeon_encoder = to_radeon_encoder(ext_encoder);
-                                        args.v3.sInput.ucExtTransmitterID = ext_radeon_encoder->encoder_id;
-                                } else
+                                if (radeon_encoder_get_dp_bridge_encoder_id(encoder) !=
+                                    ENCODER_OBJECT_ID_NONE)
+                                        args.v3.sInput.ucExtTransmitterID =
+                                                radeon_encoder_get_dp_bridge_encoder_id(encoder);
+                                else
                                         args.v3.sInput.ucExtTransmitterID = 0;
 
                                 atom_execute_table(rdev->mode_info.atom_context,
@@ -764,7 +749,7 @@ static void atombios_crtc_set_dcpll(struct drm_crtc *crtc,
 }
 
 static void atombios_crtc_program_pll(struct drm_crtc *crtc,
-                                      int crtc_id,
+                                      u32 crtc_id,
                                       int pll_id,
                                       u32 encoder_mode,
                                       u32 encoder_id,
@@ -851,8 +836,7 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
                         args.v5.ucPpll = pll_id;
                         break;
                 case 6:
-                        args.v6.ulCrtcPclkFreq.ucCRTC = crtc_id;
-                        args.v6.ulCrtcPclkFreq.ulPixelClock = cpu_to_le32(clock / 10);
+                        args.v6.ulDispEngClkFreq = cpu_to_le32(crtc_id << 24 | clock / 10);
                         args.v6.ucRefDiv = ref_div;
                         args.v6.usFbDiv = cpu_to_le16(fb_div);
                         args.v6.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000);
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index 8c0f9e36ff8..03a347a99d7 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -115,6 +115,7 @@ static int radeon_dp_aux_native_write(struct radeon_connector *radeon_connector,
         u8 msg[20];
         int msg_bytes = send_bytes + 4;
         u8 ack;
+        unsigned retry;
 
         if (send_bytes > 16)
                 return -1;
@@ -125,20 +126,22 @@ static int radeon_dp_aux_native_write(struct radeon_connector *radeon_connector,
         msg[3] = (msg_bytes << 4) | (send_bytes - 1);
         memcpy(&msg[4], send, send_bytes);
 
-        while (1) {
+        for (retry = 0; retry < 4; retry++) {
                 ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus,
                                             msg, msg_bytes, NULL, 0, delay, &ack);
-                if (ret < 0)
+                if (ret == -EBUSY)
+                        continue;
+                else if (ret < 0)
                         return ret;
                 if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK)
-                        break;
+                        return send_bytes;
                 else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER)
                         udelay(400);
                 else
                         return -EIO;
         }
 
-        return send_bytes;
+        return -EIO;
 }
 
 static int radeon_dp_aux_native_read(struct radeon_connector *radeon_connector,
@@ -149,26 +152,31 @@ static int radeon_dp_aux_native_read(struct radeon_connector *radeon_connector,
         int msg_bytes = 4;
         u8 ack;
         int ret;
+        unsigned retry;
 
         msg[0] = address;
         msg[1] = address >> 8;
         msg[2] = AUX_NATIVE_READ << 4;
         msg[3] = (msg_bytes << 4) | (recv_bytes - 1);
 
-        while (1) {
+        for (retry = 0; retry < 4; retry++) {
                 ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus,
                                             msg, msg_bytes, recv, recv_bytes, delay, &ack);
-                if (ret == 0)
-                        return -EPROTO;
-                if (ret < 0)
+                if (ret == -EBUSY)
+                        continue;
+                else if (ret < 0)
                         return ret;
                 if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK)
                         return ret;
                 else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER)
                         udelay(400);
+                else if (ret == 0)
+                        return -EPROTO;
                 else
                         return -EIO;
         }
+
+        return -EIO;
 }
 
 static void radeon_write_dpcd_reg(struct radeon_connector *radeon_connector,
@@ -232,7 +240,9 @@ int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
         for (retry = 0; retry < 4; retry++) {
                 ret = radeon_process_aux_ch(auxch,
                                             msg, msg_bytes, reply, reply_bytes, 0, &ack);
-                if (ret < 0) {
+                if (ret == -EBUSY)
+                        continue;
+                else if (ret < 0) {
                         DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
                         return ret;
                 }
@@ -273,7 +283,7 @@ int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
                 }
         }
 
-        DRM_ERROR("aux i2c too many retries, giving up\n");
+        DRM_DEBUG_KMS("aux i2c too many retries, giving up\n");
         return -EREMOTEIO;
 }
 
@@ -472,7 +482,8 @@ static int radeon_dp_get_dp_link_clock(struct drm_connector *connector,
         int bpp = convert_bpc_to_bpp(connector->display_info.bpc);
         int lane_num, max_pix_clock;
 
-        if (radeon_connector_encoder_is_dp_bridge(connector))
+        if (radeon_connector_encoder_get_dp_bridge_encoder_id(connector) !=
+            ENCODER_OBJECT_ID_NONE)
                 return 270000;
 
         lane_num = radeon_dp_get_dp_lane_number(connector, dpcd, pix_clock);
@@ -543,17 +554,32 @@ static void radeon_dp_set_panel_mode(struct drm_encoder *encoder,
 {
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         int panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE;
 
         if (!ASIC_IS_DCE4(rdev))
                 return;
 
-        if (radeon_connector_encoder_is_dp_bridge(connector))
+        if (radeon_connector_encoder_get_dp_bridge_encoder_id(connector) ==
+            ENCODER_OBJECT_ID_NUTMEG)
                 panel_mode = DP_PANEL_MODE_INTERNAL_DP1_MODE;
+        else if (radeon_connector_encoder_get_dp_bridge_encoder_id(connector) ==
+                 ENCODER_OBJECT_ID_TRAVIS)
+                panel_mode = DP_PANEL_MODE_INTERNAL_DP2_MODE;
+        else if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
+                u8 tmp = radeon_read_dpcd_reg(radeon_connector, DP_EDP_CONFIGURATION_CAP);
+                if (tmp & 1)
+                        panel_mode = DP_PANEL_MODE_INTERNAL_DP2_MODE;
+        }
 
         atombios_dig_encoder_setup(encoder,
                                    ATOM_ENCODER_CMD_SETUP_PANEL_MODE,
                                    panel_mode);
+
+        if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) &&
+            (panel_mode == DP_PANEL_MODE_INTERNAL_DP2_MODE)) {
+                radeon_write_dpcd_reg(radeon_connector, DP_EDP_CONFIGURATION_SET, 1);
+        }
 }
 
 void radeon_dp_set_link_config(struct drm_connector *connector,
@@ -613,6 +639,18 @@ static bool radeon_dp_get_link_status(struct radeon_connector *radeon_connector,
         return true;
 }
 
+bool radeon_dp_needs_link_train(struct radeon_connector *radeon_connector)
+{
+        u8 link_status[DP_LINK_STATUS_SIZE];
+        struct radeon_connector_atom_dig *dig = radeon_connector->con_priv;
+
+        if (!radeon_dp_get_link_status(radeon_connector, link_status))
+                return false;
+        if (dp_channel_eq_ok(link_status, dig->dp_lane_count))
+                return false;
+        return true;
+}
+
 struct radeon_dp_link_train_info {
         struct radeon_device *rdev;
         struct drm_encoder *encoder;
@@ -627,6 +665,7 @@ struct radeon_dp_link_train_info {
         u8 train_set[4];
         u8 link_status[DP_LINK_STATUS_SIZE];
         u8 tries;
+        bool use_dpencoder;
 };
 
 static void radeon_dp_update_vs_emph(struct radeon_dp_link_train_info *dp_info)
@@ -646,7 +685,7 @@ static void radeon_dp_set_tp(struct radeon_dp_link_train_info *dp_info, int tp)
         int rtp = 0;
 
         /* set training pattern on the source */
-        if (ASIC_IS_DCE4(dp_info->rdev)) {
+        if (ASIC_IS_DCE4(dp_info->rdev) || !dp_info->use_dpencoder) {
                 switch (tp) {
                 case DP_TRAINING_PATTERN_1:
                         rtp = ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1;
@@ -706,7 +745,7 @@ static int radeon_dp_link_train_init(struct radeon_dp_link_train_info *dp_info)
         radeon_write_dpcd_reg(dp_info->radeon_connector, DP_LINK_BW_SET, tmp);
 
         /* start training on the source */
-        if (ASIC_IS_DCE4(dp_info->rdev))
+        if (ASIC_IS_DCE4(dp_info->rdev) || !dp_info->use_dpencoder)
                 atombios_dig_encoder_setup(dp_info->encoder,
                                            ATOM_ENCODER_CMD_DP_LINK_TRAINING_START, 0);
         else
@@ -731,7 +770,7 @@ static int radeon_dp_link_train_finish(struct radeon_dp_link_train_info *dp_info
                               DP_TRAINING_PATTERN_DISABLE);
 
         /* disable the training pattern on the source */
-        if (ASIC_IS_DCE4(dp_info->rdev))
+        if (ASIC_IS_DCE4(dp_info->rdev) || !dp_info->use_dpencoder)
                 atombios_dig_encoder_setup(dp_info->encoder,
                                            ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE, 0);
         else
@@ -869,7 +908,8 @@ void radeon_dp_link_train(struct drm_encoder *encoder,
         struct radeon_connector *radeon_connector;
         struct radeon_connector_atom_dig *dig_connector;
         struct radeon_dp_link_train_info dp_info;
-        u8 tmp;
+        int index;
+        u8 tmp, frev, crev;
 
         if (!radeon_encoder->enc_priv)
                 return;
@@ -884,6 +924,18 @@ void radeon_dp_link_train(struct drm_encoder *encoder,
             (dig_connector->dp_sink_type != CONNECTOR_OBJECT_ID_eDP))
                 return;
 
+        /* DPEncoderService newer than 1.1 can't program properly the
+         * training pattern. When facing such version use the
+         * DIGXEncoderControl (X== 1 | 2)
+         */
+        dp_info.use_dpencoder = true;
+        index = GetIndexIntoMasterTable(COMMAND, DPEncoderService);
+        if (atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev)) {
+                if (crev > 1) {
+                        dp_info.use_dpencoder = false;
+                }
+        }
+
         dp_info.enc_id = 0;
         if (dig->dig_encoder)
                 dp_info.enc_id |= ATOM_DP_CONFIG_DIG2_ENCODER;
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 15bd0477a3e..c3f0d428c51 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -41,6 +41,31 @@ static void evergreen_gpu_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
 static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 
+void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
+{
+        u16 ctl, v;
+        int cap, err;
+
+        cap = pci_pcie_cap(rdev->pdev);
+        if (!cap)
+                return;
+
+        err = pci_read_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, &ctl);
+        if (err)
+                return;
+
+        v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
+
+        /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
+         * to avoid hangs or perfomance issues
+         */
+        if ((v == 0) || (v == 6) || (v == 7)) {
+                ctl &= ~PCI_EXP_DEVCTL_READRQ;
+                ctl |= (2 << 12);
+                pci_write_config_word(rdev->pdev, cap + PCI_EXP_DEVCTL, ctl);
+        }
+}
+
 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
 {
         /* enable the pflip int */
@@ -57,6 +82,7 @@ u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 {
         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
+        int i;
 
         /* Lock the graphics update lock */
         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
@@ -74,7 +100,11 @@ u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
                (u32)crtc_base);
 
         /* Wait for update_pending to go high. */
-        while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
+                        break;
+                udelay(1);
+        }
         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 
         /* Unlock the lock, so double-buffering can take place inside vblank */
@@ -328,6 +358,7 @@ void evergreen_hpd_init(struct radeon_device *rdev)
                 default:
                         break;
                 }
+                radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
         }
         if (rdev->irq.installed)
                 evergreen_irq_set(rdev);
@@ -743,7 +774,7 @@ static void evergreen_program_watermarks(struct radeon_device *rdev,
                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
                     !evergreen_check_latency_hiding(&wm) ||
                     (rdev->disp_priority == 2)) {
-                        DRM_INFO("force priority to high\n");
+                        DRM_DEBUG_KMS("force priority to high\n");
                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
                 }
@@ -1357,6 +1388,7 @@ int evergreen_cp_resume(struct radeon_device *rdev)
                                  SOFT_RESET_PA |
                                  SOFT_RESET_SH |
                                  SOFT_RESET_VGT |
+                                 SOFT_RESET_SPI |
                                  SOFT_RESET_SX));
         RREG32(GRBM_SOFT_RESET);
         mdelay(15);
@@ -1378,13 +1410,11 @@ int evergreen_cp_resume(struct radeon_device *rdev)
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
         WREG32(CP_RB_RPTR_WR, 0);
-        WREG32(CP_RB_WPTR, 0);
+        rdev->cp.wptr = 0;
+        WREG32(CP_RB_WPTR, rdev->cp.wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB_RPTR_ADDR,
-#ifdef __BIG_ENDIAN
-               RB_RPTR_SWAP(2) |
-#endif
                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
@@ -1403,7 +1433,6 @@ int evergreen_cp_resume(struct radeon_device *rdev)
         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
 
         rdev->cp.rptr = RREG32(CP_RB_RPTR);
-        rdev->cp.wptr = RREG32(CP_RB_WPTR);
 
         evergreen_cp_start(rdev);
         rdev->cp.ready = true;
@@ -1567,48 +1596,6 @@ static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         return backend_map;
 }
 
-static void evergreen_program_channel_remap(struct radeon_device *rdev)
-{
-        u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
-
-        tmp = RREG32(MC_SHARED_CHMAP);
-        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
-        case 0:
-        case 1:
-        case 2:
-        case 3:
-        default:
-                /* default mapping */
-                mc_shared_chremap = 0x00fac688;
-                break;
-        }
-
-        switch (rdev->family) {
-        case CHIP_HEMLOCK:
-        case CHIP_CYPRESS:
-        case CHIP_BARTS:
-                tcp_chan_steer_lo = 0x54763210;
-                tcp_chan_steer_hi = 0x0000ba98;
-                break;
-        case CHIP_JUNIPER:
-        case CHIP_REDWOOD:
-        case CHIP_CEDAR:
-        case CHIP_PALM:
-        case CHIP_SUMO:
-        case CHIP_SUMO2:
-        case CHIP_TURKS:
-        case CHIP_CAICOS:
-        default:
-                tcp_chan_steer_lo = 0x76543210;
-                tcp_chan_steer_hi = 0x0000ba98;
-                break;
-        }
-
-        WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
-        WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
-        WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
-}
-
 static void evergreen_gpu_init(struct radeon_device *rdev)
 {
         u32 cc_rb_backend_disable = 0;
@@ -1865,6 +1852,8 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
 
         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 
+        evergreen_fix_pci_max_read_req_size(rdev);
+
         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
 
         cc_gc_shader_pipe_config |=
@@ -2047,13 +2036,12 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
         rdev->config.evergreen.tile_config |=
                 ((gb_addr_config & 0x30000000) >> 28) << 12;
 
+        rdev->config.evergreen.backend_map = gb_backend_map;
         WREG32(GB_BACKEND_MAP, gb_backend_map);
         WREG32(GB_ADDR_CONFIG, gb_addr_config);
         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
 
-        evergreen_program_channel_remap(rdev);
-
         num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
         grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
 
@@ -2761,6 +2749,9 @@ int evergreen_irq_process(struct radeon_device *rdev)
                 return IRQ_NONE;
         }
 restart_ih:
+        /* Order reading of wptr vs. reading of IH ring data */
+        rmb();
+
         /* display interrupts */
         evergreen_irq_ack(rdev);
 
@@ -3142,21 +3133,23 @@ int evergreen_suspend(struct radeon_device *rdev)
 }
 
 int evergreen_copy_blit(struct radeon_device *rdev,
-                        uint64_t src_offset, uint64_t dst_offset,
-                        unsigned num_pages, struct radeon_fence *fence)
+                        uint64_t src_offset,
+                        uint64_t dst_offset,
+                        unsigned num_gpu_pages,
+                        struct radeon_fence *fence)
 {
         int r;
 
         mutex_lock(&rdev->r600_blit.mutex);
         rdev->r600_blit.vb_ib = NULL;
-        r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
+        r = evergreen_blit_prepare_copy(rdev, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
         if (r) {
                 if (rdev->r600_blit.vb_ib)
                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
                 mutex_unlock(&rdev->r600_blit.mutex);
                 return r;
         }
-        evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
+        evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
         evergreen_blit_done_copy(rdev, fence);
         mutex_unlock(&rdev->r600_blit.mutex);
         return 0;
@@ -3265,6 +3258,18 @@ int evergreen_init(struct radeon_device *rdev)
                         rdev->accel_working = false;
                 }
         }
+
+        /* Don't start up if the MC ucode is missing on BTC parts.
+         * The default clocks and voltages before the MC ucode
+         * is loaded are not suffient for advanced operations.
+         */
+        if (ASIC_IS_DCE5(rdev)) {
+                if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
+                        DRM_ERROR("radeon: MC ucode required for NI+.\n");
+                        return -EINVAL;
+                }
+        }
+
         return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index 23d36417158..a134790903d 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -428,7 +428,7 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
                 last_reg = ARRAY_SIZE(evergreen_reg_safe_bm);
 
         i = (reg >> 7);
-        if (i > last_reg) {
+        if (i >= last_reg) {
                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                 return -EINVAL;
         }
@@ -856,7 +856,6 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
         case SQ_PGM_START_PS:
         case SQ_PGM_START_HS:
         case SQ_PGM_START_LS:
-        case GDS_ADDR_BASE:
         case SQ_CONST_MEM_BASE:
         case SQ_ALU_CONST_CACHE_GS_0:
         case SQ_ALU_CONST_CACHE_GS_1:
@@ -946,6 +945,34 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
                 }
                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                 break;
+        case SX_MEMORY_EXPORT_BASE:
+                if (p->rdev->family >= CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONFIG_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONFIG_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                break;
+        case CAYMAN_SX_SCATTER_EXPORT_BASE:
+                if (p->rdev->family < CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                break;
         default:
                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                 return -EINVAL;
@@ -1153,6 +1180,34 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 break;
+        case PACKET3_DISPATCH_DIRECT:
+                if (pkt->count != 3) {
+                        DRM_ERROR("bad DISPATCH_DIRECT\n");
+                        return -EINVAL;
+                }
+                r = evergreen_cs_track_check(p);
+                if (r) {
+                        dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
+                        return r;
+                }
+                break;
+        case PACKET3_DISPATCH_INDIRECT:
+                if (pkt->count != 1) {
+                        DRM_ERROR("bad DISPATCH_INDIRECT\n");
+                        return -EINVAL;
+                }
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        DRM_ERROR("bad DISPATCH_INDIRECT\n");
+                        return -EINVAL;
+                }
+                ib[idx+0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
+                r = evergreen_cs_track_check(p);
+                if (r) {
+                        dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
+                        return r;
+                }
+                break;
         case PACKET3_WAIT_REG_MEM:
                 if (pkt->count != 5) {
                         DRM_ERROR("bad WAIT_REG_MEM\n");
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index b7b2714f0b3..7363d9dec90 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -351,6 +351,7 @@
 #define         COLOR_BUFFER_SIZE(x)                            ((x) << 0)
 #define         POSITION_BUFFER_SIZE(x)                         ((x) << 8)
 #define         SMX_BUFFER_SIZE(x)                              ((x) << 16)
+#define SX_MEMORY_EXPORT_BASE                           0x9010
 #define SX_MISC                                         0x28350
 
 #define CB_PERF_CTR0_SEL_0                              0x9A20
@@ -1122,6 +1123,7 @@
 #define CAYMAN_PA_SC_AA_CONFIG                          0x28BE0
 #define         CAYMAN_MSAA_NUM_SAMPLES_SHIFT           0
 #define         CAYMAN_MSAA_NUM_SAMPLES_MASK            0x7
+#define CAYMAN_SX_SCATTER_EXPORT_BASE                   0x28358
 /* cayman packet3 addition */
 #define CAYMAN_PACKET3_DEALLOC_STATE                    0x14
 
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 559dbd41290..8c79ca97753 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -39,6 +39,7 @@ extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 extern void evergreen_mc_program(struct radeon_device *rdev);
 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 extern int evergreen_mc_init(struct radeon_device *rdev);
+extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 
 #define EVERGREEN_PFP_UCODE_SIZE 1120
 #define EVERGREEN_PM4_UCODE_SIZE 1376
@@ -568,36 +569,6 @@ static u32 cayman_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         return backend_map;
 }
 
-static void cayman_program_channel_remap(struct radeon_device *rdev)
-{
-        u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
-
-        tmp = RREG32(MC_SHARED_CHMAP);
-        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
-        case 0:
-        case 1:
-        case 2:
-        case 3:
-        default:
-                /* default mapping */
-                mc_shared_chremap = 0x00fac688;
-                break;
-        }
-
-        switch (rdev->family) {
-        case CHIP_CAYMAN:
-        default:
-                //tcp_chan_steer_lo = 0x54763210
-                tcp_chan_steer_lo = 0x76543210;
-                tcp_chan_steer_hi = 0x0000ba98;
-                break;
-        }
-
-        WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
-        WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
-        WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
-}
-
 static u32 cayman_get_disable_mask_per_asic(struct radeon_device *rdev,
                                             u32 disable_mask_per_se,
                                             u32 max_disable_mask_per_se,
@@ -669,6 +640,8 @@ static void cayman_gpu_init(struct radeon_device *rdev)
 
         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 
+        evergreen_fix_pci_max_read_req_size(rdev);
+
         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 
@@ -833,13 +806,12 @@ static void cayman_gpu_init(struct radeon_device *rdev)
         rdev->config.cayman.tile_config |=
                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
 
+        rdev->config.cayman.backend_map = gb_backend_map;
         WREG32(GB_BACKEND_MAP, gb_backend_map);
         WREG32(GB_ADDR_CONFIG, gb_addr_config);
         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
 
-        cayman_program_channel_remap(rdev);
-
         /* primary versions */
         WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
         WREG32(CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
@@ -1158,6 +1130,7 @@ int cayman_cp_resume(struct radeon_device *rdev)
                                  SOFT_RESET_PA |
                                  SOFT_RESET_SH |
                                  SOFT_RESET_VGT |
+                                 SOFT_RESET_SPI |
                                  SOFT_RESET_SX));
         RREG32(GRBM_SOFT_RESET);
         mdelay(15);
@@ -1182,7 +1155,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
-        WREG32(CP_RB0_WPTR, 0);
+        rdev->cp.wptr = 0;
+        WREG32(CP_RB0_WPTR, rdev->cp.wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1202,7 +1176,6 @@ int cayman_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB0_BASE, rdev->cp.gpu_addr >> 8);
 
         rdev->cp.rptr = RREG32(CP_RB0_RPTR);
-        rdev->cp.wptr = RREG32(CP_RB0_WPTR);
 
         /* ring1  - compute only */
         /* Set ring buffer size */
@@ -1215,7 +1188,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
-        WREG32(CP_RB1_WPTR, 0);
+        rdev->cp1.wptr = 0;
+        WREG32(CP_RB1_WPTR, rdev->cp1.wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1227,7 +1201,6 @@ int cayman_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB1_BASE, rdev->cp1.gpu_addr >> 8);
 
         rdev->cp1.rptr = RREG32(CP_RB1_RPTR);
-        rdev->cp1.wptr = RREG32(CP_RB1_WPTR);
 
         /* ring2 - compute only */
         /* Set ring buffer size */
@@ -1240,7 +1213,8 @@ int cayman_cp_resume(struct radeon_device *rdev)
 
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
-        WREG32(CP_RB2_WPTR, 0);
+        rdev->cp2.wptr = 0;
+        WREG32(CP_RB2_WPTR, rdev->cp2.wptr);
 
         /* set the wb address wether it's enabled or not */
         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
@@ -1252,7 +1226,6 @@ int cayman_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB2_BASE, rdev->cp2.gpu_addr >> 8);
 
         rdev->cp2.rptr = RREG32(CP_RB2_RPTR);
-        rdev->cp2.wptr = RREG32(CP_RB2_WPTR);
 
         /* start the rings */
         cayman_cp_start(rdev);
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index f2204cb1ccd..764249587f1 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -84,13 +84,18 @@ u32 r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 {
         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
         u32 tmp = ((u32)crtc_base) | RADEON_CRTC_OFFSET__OFFSET_LOCK;
+        int i;
 
         /* Lock the graphics update lock */
         /* update the scanout addresses */
         WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp);
 
         /* Wait for update_pending to go high. */
-        while (!(RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET));
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET)
+                        break;
+                udelay(1);
+        }
         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 
         /* Unlock the lock, so double-buffering can take place inside vblank */
@@ -434,6 +439,7 @@ void r100_hpd_init(struct radeon_device *rdev)
                 default:
                         break;
                 }
+                radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
         }
         if (rdev->irq.installed)
                 r100_irq_set(rdev);
@@ -721,11 +727,11 @@ void r100_fence_ring_emit(struct radeon_device *rdev,
 int r100_copy_blit(struct radeon_device *rdev,
                    uint64_t src_offset,
                    uint64_t dst_offset,
-                   unsigned num_pages,
+                   unsigned num_gpu_pages,
                    struct radeon_fence *fence)
 {
         uint32_t cur_pages;
-        uint32_t stride_bytes = PAGE_SIZE;
+        uint32_t stride_bytes = RADEON_GPU_PAGE_SIZE;
         uint32_t pitch;
         uint32_t stride_pixels;
         unsigned ndw;
@@ -737,7 +743,7 @@ int r100_copy_blit(struct radeon_device *rdev,
         /* radeon pitch is /64 */
         pitch = stride_bytes / 64;
         stride_pixels = stride_bytes / 4;
-        num_loops = DIV_ROUND_UP(num_pages, 8191);
+        num_loops = DIV_ROUND_UP(num_gpu_pages, 8191);
 
         /* Ask for enough room for blit + flush + fence */
         ndw = 64 + (10 * num_loops);
@@ -746,12 +752,12 @@ int r100_copy_blit(struct radeon_device *rdev,
                 DRM_ERROR("radeon: moving bo (%d) asking for %u dw.\n", r, ndw);
                 return -EINVAL;
         }
-        while (num_pages > 0) {
-                cur_pages = num_pages;
+        while (num_gpu_pages > 0) {
+                cur_pages = num_gpu_pages;
                 if (cur_pages > 8191) {
                         cur_pages = 8191;
                 }
-                num_pages -= cur_pages;
+                num_gpu_pages -= cur_pages;
 
                 /* pages are in Y direction - height
                    page width in X direction - width */
@@ -773,8 +779,8 @@ int r100_copy_blit(struct radeon_device *rdev,
                 radeon_ring_write(rdev, (0x1fff) | (0x1fff << 16));
                 radeon_ring_write(rdev, 0);
                 radeon_ring_write(rdev, (0x1fff) | (0x1fff << 16));
-                radeon_ring_write(rdev, num_pages);
-                radeon_ring_write(rdev, num_pages);
+                radeon_ring_write(rdev, num_gpu_pages);
+                radeon_ring_write(rdev, num_gpu_pages);
                 radeon_ring_write(rdev, cur_pages | (stride_pixels << 16));
         }
         radeon_ring_write(rdev, PACKET0(RADEON_DSTCACHE_CTLSTAT, 0));
@@ -990,7 +996,8 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
         /* Force read & write ptr to 0 */
         WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA | RADEON_RB_NO_UPDATE);
         WREG32(RADEON_CP_RB_RPTR_WR, 0);
-        WREG32(RADEON_CP_RB_WPTR, 0);
+        rdev->cp.wptr = 0;
+        WREG32(RADEON_CP_RB_WPTR, rdev->cp.wptr);
 
         /* set the wb address whether it's enabled or not */
         WREG32(R_00070C_CP_RB_RPTR_ADDR,
@@ -1007,9 +1014,6 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
         WREG32(RADEON_CP_RB_CNTL, tmp);
         udelay(10);
         rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
-        rdev->cp.wptr = RREG32(RADEON_CP_RB_WPTR);
-        /* protect against crazy HW on resume */
-        rdev->cp.wptr &= rdev->cp.ptr_mask;
         /* Set cp mode to bus mastering & enable cp*/
         WREG32(RADEON_CP_CSQ_MODE,
                REG_SET(RADEON_INDIRECT2_START, indirect2_start) |
@@ -2065,6 +2069,7 @@ bool r100_gpu_is_lockup(struct radeon_device *rdev)
 void r100_bm_disable(struct radeon_device *rdev)
 {
         u32 tmp;
+        u16 tmp16;
 
         /* disable bus mastering */
         tmp = RREG32(R_000030_BUS_CNTL);
@@ -2075,8 +2080,8 @@ void r100_bm_disable(struct radeon_device *rdev)
         WREG32(R_000030_BUS_CNTL, (tmp & 0xFFFFFFFF) | 0x00000040);
         tmp = RREG32(RADEON_BUS_CNTL);
         mdelay(1);
-        pci_read_config_word(rdev->pdev, 0x4, (u16*)&tmp);
-        pci_write_config_word(rdev->pdev, 0x4, tmp & 0xFFFB);
+        pci_read_config_word(rdev->pdev, 0x4, &tmp16);
+        pci_write_config_word(rdev->pdev, 0x4, tmp16 & 0xFFFB);
         mdelay(1);
 }
 
diff --git a/drivers/gpu/drm/radeon/r200.c b/drivers/gpu/drm/radeon/r200.c
index f2405830041..a1f3ba063c2 100644
--- a/drivers/gpu/drm/radeon/r200.c
+++ b/drivers/gpu/drm/radeon/r200.c
@@ -84,7 +84,7 @@ static int r200_get_vtx_size_0(uint32_t vtx_fmt_0)
 int r200_copy_dma(struct radeon_device *rdev,
                   uint64_t src_offset,
                   uint64_t dst_offset,
-                  unsigned num_pages,
+                  unsigned num_gpu_pages,
                   struct radeon_fence *fence)
 {
         uint32_t size;
@@ -93,7 +93,7 @@ int r200_copy_dma(struct radeon_device *rdev,
         int r = 0;
 
         /* radeon pitch is /64 */
-        size = num_pages << PAGE_SHIFT;
+        size = num_gpu_pages << RADEON_GPU_PAGE_SHIFT;
         num_loops = DIV_ROUND_UP(size, 0x1FFFFF);
         r = radeon_ring_lock(rdev, num_loops * 4 + 64);
         if (r) {
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index bc54b26cb32..9b62a97742a 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -762,13 +762,14 @@ void r600_hpd_init(struct radeon_device *rdev)
         struct drm_device *dev = rdev->ddev;
         struct drm_connector *connector;
 
-        if (ASIC_IS_DCE3(rdev)) {
-                u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
-                if (ASIC_IS_DCE32(rdev))
-                        tmp |= DC_HPDx_EN;
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+
+                if (ASIC_IS_DCE3(rdev)) {
+                        u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
+                        if (ASIC_IS_DCE32(rdev))
+                                tmp |= DC_HPDx_EN;
 
-                list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
                         switch (radeon_connector->hpd.hpd) {
                         case RADEON_HPD_1:
                                 WREG32(DC_HPD1_CONTROL, tmp);
@@ -798,10 +799,7 @@ void r600_hpd_init(struct radeon_device *rdev)
                         default:
                                 break;
                         }
-                }
-        } else {
-                list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                } else {
                         switch (radeon_connector->hpd.hpd) {
                         case RADEON_HPD_1:
                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
@@ -819,6 +817,7 @@ void r600_hpd_init(struct radeon_device *rdev)
                                 break;
                         }
                 }
+                radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
         }
         if (rdev->irq.installed)
                 r600_irq_set(rdev);
@@ -1662,6 +1661,7 @@ void r600_gpu_init(struct radeon_device *rdev)
                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
                                                         (cc_rb_backend_disable >> 16));
         rdev->config.r600.tile_config = tiling_config;
+        rdev->config.r600.backend_map = backend_map;
         tiling_config |= BACKEND_MAP(backend_map);
         WREG32(GB_TILING_CONFIG, tiling_config);
         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
@@ -2208,13 +2208,11 @@ int r600_cp_resume(struct radeon_device *rdev)
         /* Initialize the ring buffer's read and write pointers */
         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
         WREG32(CP_RB_RPTR_WR, 0);
-        WREG32(CP_RB_WPTR, 0);
+        rdev->cp.wptr = 0;
+        WREG32(CP_RB_WPTR, rdev->cp.wptr);
 
         /* set the wb address whether it's enabled or not */
         WREG32(CP_RB_RPTR_ADDR,
-#ifdef __BIG_ENDIAN
-               RB_RPTR_SWAP(2) |
-#endif
                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
@@ -2233,7 +2231,6 @@ int r600_cp_resume(struct radeon_device *rdev)
         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
 
         rdev->cp.rptr = RREG32(CP_RB_RPTR);
-        rdev->cp.wptr = RREG32(CP_RB_WPTR);
 
         r600_cp_start(rdev);
         rdev->cp.ready = true;
@@ -2355,21 +2352,23 @@ void r600_fence_ring_emit(struct radeon_device *rdev,
 }
 
 int r600_copy_blit(struct radeon_device *rdev,
-                   uint64_t src_offset, uint64_t dst_offset,
-                   unsigned num_pages, struct radeon_fence *fence)
+                   uint64_t src_offset,
+                   uint64_t dst_offset,
+                   unsigned num_gpu_pages,
+                   struct radeon_fence *fence)
 {
         int r;
 
         mutex_lock(&rdev->r600_blit.mutex);
         rdev->r600_blit.vb_ib = NULL;
-        r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
+        r = r600_blit_prepare_copy(rdev, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
         if (r) {
                 if (rdev->r600_blit.vb_ib)
                         radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
                 mutex_unlock(&rdev->r600_blit.mutex);
                 return r;
         }
-        r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
+        r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages * RADEON_GPU_PAGE_SIZE);
         r600_blit_done_copy(rdev, fence);
         mutex_unlock(&rdev->r600_blit.mutex);
         return 0;
@@ -2994,10 +2993,6 @@ int r600_irq_init(struct radeon_device *rdev)
         /* RPTR_REARM only works if msi's are enabled */
         if (rdev->msi_enabled)
                 ih_cntl |= RPTR_REARM;
-
-#ifdef __BIG_ENDIAN
-        ih_cntl |= IH_MC_SWAP(IH_MC_SWAP_32BIT);
-#endif
         WREG32(IH_CNTL, ih_cntl);
 
         /* force the active interrupt state to all disabled */
@@ -3308,6 +3303,10 @@ int r600_irq_process(struct radeon_device *rdev)
         if (!rdev->ih.enabled || rdev->shutdown)
                 return IRQ_NONE;
 
+        /* No MSIs, need a dummy read to flush PCI DMAs */
+        if (!rdev->msi_enabled)
+                RREG32(IH_RB_WPTR);
+
         wptr = r600_get_ih_wptr(rdev);
         rptr = rdev->ih.rptr;
         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
@@ -3320,6 +3319,9 @@ int r600_irq_process(struct radeon_device *rdev)
         }
 
 restart_ih:
+        /* Order reading of wptr vs. reading of IH ring data */
+        rmb();
+
         /* display interrupts */
         r600_irq_ack(rdev);
 
diff --git a/drivers/gpu/drm/radeon/r600_cp.c b/drivers/gpu/drm/radeon/r600_cp.c
index c3ab959bdc7..45fd592f960 100644
--- a/drivers/gpu/drm/radeon/r600_cp.c
+++ b/drivers/gpu/drm/radeon/r600_cp.c
@@ -1802,8 +1802,8 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
         /* Set ring buffer size */
 #ifdef __BIG_ENDIAN
         RADEON_WRITE(R600_CP_RB_CNTL,
-                     RADEON_BUF_SWAP_32BIT |
-                     RADEON_RB_NO_UPDATE |
+                     R600_BUF_SWAP_32BIT |
+                     R600_RB_NO_UPDATE |
                      (dev_priv->ring.rptr_update_l2qw << 8) |
                      dev_priv->ring.size_l2qw);
 #else
@@ -1820,15 +1820,15 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
 
 #ifdef __BIG_ENDIAN
         RADEON_WRITE(R600_CP_RB_CNTL,
-                     RADEON_BUF_SWAP_32BIT |
-                     RADEON_RB_NO_UPDATE |
-                     RADEON_RB_RPTR_WR_ENA |
+                     R600_BUF_SWAP_32BIT |
+                     R600_RB_NO_UPDATE |
+                     R600_RB_RPTR_WR_ENA |
                      (dev_priv->ring.rptr_update_l2qw << 8) |
                      dev_priv->ring.size_l2qw);
 #else
         RADEON_WRITE(R600_CP_RB_CNTL,
-                     RADEON_RB_NO_UPDATE |
-                     RADEON_RB_RPTR_WR_ENA |
+                     R600_RB_NO_UPDATE |
+                     R600_RB_RPTR_WR_ENA |
                      (dev_priv->ring.rptr_update_l2qw << 8) |
                      dev_priv->ring.size_l2qw);
 #endif
@@ -1851,13 +1851,8 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
                         - ((unsigned long) dev->sg->virtual)
                         + dev_priv->gart_vm_start;
         }
-        RADEON_WRITE(R600_CP_RB_RPTR_ADDR,
-#ifdef __BIG_ENDIAN
-                     (2 << 0) |
-#endif
-                     (rptr_addr & 0xfffffffc));
-        RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI,
-                     upper_32_bits(rptr_addr));
+        RADEON_WRITE(R600_CP_RB_RPTR_ADDR, (rptr_addr & 0xfffffffc));
+        RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI, upper_32_bits(rptr_addr));
 
 #ifdef __BIG_ENDIAN
         RADEON_WRITE(R600_CP_RB_CNTL,
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 909bda8dd55..cf83aa05a68 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -915,12 +915,11 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
 {
         struct r600_cs_track *track = (struct r600_cs_track *)p->track;
         struct radeon_cs_reloc *reloc;
-        u32 last_reg = ARRAY_SIZE(r600_reg_safe_bm);
         u32 m, i, tmp, *ib;
         int r;
 
         i = (reg >> 7);
-        if (i > last_reg) {
+        if (i >= ARRAY_SIZE(r600_reg_safe_bm)) {
                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                 return -EINVAL;
         }
@@ -1200,6 +1199,15 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
                 }
                 ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                 break;
+        case SX_MEMORY_EXPORT_BASE:
+                r = r600_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONFIG_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                break;
         default:
                 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                 return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/r600_hdmi.c b/drivers/gpu/drm/radeon/r600_hdmi.c
index f5ac7e788d8..c45d92191fd 100644
--- a/drivers/gpu/drm/radeon/r600_hdmi.c
+++ b/drivers/gpu/drm/radeon/r600_hdmi.c
@@ -196,6 +196,13 @@ static void r600_hdmi_videoinfoframe(
         frame[0xD] = (right_bar >> 8);
 
         r600_hdmi_infoframe_checksum(0x82, 0x02, 0x0D, frame);
+        /* Our header values (type, version, length) should be alright, Intel
+         * is using the same. Checksum function also seems to be OK, it works
+         * fine for audio infoframe. However calculated value is always lower
+         * by 2 in comparison to fglrx. It breaks displaying anything in case
+         * of TVs that strictly check the checksum. Hack it manually here to
+         * workaround this issue. */
+        frame[0x0] += 2;
 
         WREG32(offset+R600_HDMI_VIDEOINFOFRAME_0,
                 frame[0x0] | (frame[0x1] << 8) | (frame[0x2] << 16) | (frame[0x3] << 24));
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index ef0e0e01691..184628c7e02 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -60,7 +60,7 @@
  *                          are considered as fatal)
  */
 
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 #include <linux/wait.h>
 #include <linux/list.h>
 #include <linux/kref.h>
@@ -93,6 +93,7 @@ extern int radeon_audio;
 extern int radeon_disp_priority;
 extern int radeon_hw_i2c;
 extern int radeon_pcie_gen2;
+extern int radeon_msi;
 
 /*
  * Copy from radeon_drv.h so we don't have to include both and have conflicting
@@ -322,6 +323,7 @@ union radeon_gart_table {
 
 #define RADEON_GPU_PAGE_SIZE 4096
 #define RADEON_GPU_PAGE_MASK (RADEON_GPU_PAGE_SIZE - 1)
+#define RADEON_GPU_PAGE_SHIFT 12
 
 struct radeon_gart {
         dma_addr_t                      table_addr;
@@ -914,17 +916,17 @@ struct radeon_asic {
         int (*copy_blit)(struct radeon_device *rdev,
                          uint64_t src_offset,
                          uint64_t dst_offset,
-                         unsigned num_pages,
+                         unsigned num_gpu_pages,
                          struct radeon_fence *fence);
         int (*copy_dma)(struct radeon_device *rdev,
                         uint64_t src_offset,
                         uint64_t dst_offset,
-                        unsigned num_pages,
+                        unsigned num_gpu_pages,
                         struct radeon_fence *fence);
         int (*copy)(struct radeon_device *rdev,
                     uint64_t src_offset,
                     uint64_t dst_offset,
-                    unsigned num_pages,
+                    unsigned num_gpu_pages,
                     struct radeon_fence *fence);
         uint32_t (*get_engine_clock)(struct radeon_device *rdev);
         void (*set_engine_clock)(struct radeon_device *rdev, uint32_t eng_clock);
@@ -1003,6 +1005,7 @@ struct r600_asic {
         unsigned                tiling_npipes;
         unsigned                tiling_group_size;
         unsigned                tile_config;
+        unsigned                backend_map;
         struct r100_gpu_lockup  lockup;
 };
 
@@ -1028,6 +1031,7 @@ struct rv770_asic {
         unsigned                tiling_npipes;
         unsigned                tiling_group_size;
         unsigned                tile_config;
+        unsigned                backend_map;
         struct r100_gpu_lockup  lockup;
 };
 
@@ -1054,6 +1058,7 @@ struct evergreen_asic {
         unsigned tiling_npipes;
         unsigned tiling_group_size;
         unsigned tile_config;
+        unsigned backend_map;
         struct r100_gpu_lockup  lockup;
 };
 
@@ -1174,7 +1179,7 @@ struct radeon_device {
         /* Register mmio */
         resource_size_t                 rmmio_base;
         resource_size_t                 rmmio_size;
-        void                            *rmmio;
+        void __iomem                    *rmmio;
         radeon_rreg_t                   mc_rreg;
         radeon_wreg_t                   mc_wreg;
         radeon_rreg_t                   pll_rreg;
@@ -1251,20 +1256,20 @@ int radeon_gpu_wait_for_idle(struct radeon_device *rdev);
 static inline uint32_t r100_mm_rreg(struct radeon_device *rdev, uint32_t reg)
 {
         if (reg < rdev->rmmio_size)
-                return readl(((void __iomem *)rdev->rmmio) + reg);
+                return readl((rdev->rmmio) + reg);
         else {
-                writel(reg, ((void __iomem *)rdev->rmmio) + RADEON_MM_INDEX);
-                return readl(((void __iomem *)rdev->rmmio) + RADEON_MM_DATA);
+                writel(reg, (rdev->rmmio) + RADEON_MM_INDEX);
+                return readl((rdev->rmmio) + RADEON_MM_DATA);
         }
 }
 
 static inline void r100_mm_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
 {
         if (reg < rdev->rmmio_size)
-                writel(v, ((void __iomem *)rdev->rmmio) + reg);
+                writel(v, (rdev->rmmio) + reg);
         else {
-                writel(reg, ((void __iomem *)rdev->rmmio) + RADEON_MM_INDEX);
-                writel(v, ((void __iomem *)rdev->rmmio) + RADEON_MM_DATA);
+                writel(reg, (rdev->rmmio) + RADEON_MM_INDEX);
+                writel(v, (rdev->rmmio) + RADEON_MM_DATA);
         }
 }
 
@@ -1296,10 +1301,10 @@ static inline void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 /*
  * Registers read & write functions.
  */
-#define RREG8(reg) readb(((void __iomem *)rdev->rmmio) + (reg))
-#define WREG8(reg, v) writeb(v, ((void __iomem *)rdev->rmmio) + (reg))
-#define RREG16(reg) readw(((void __iomem *)rdev->rmmio) + (reg))
-#define WREG16(reg, v) writew(v, ((void __iomem *)rdev->rmmio) + (reg))
+#define RREG8(reg) readb((rdev->rmmio) + (reg))
+#define WREG8(reg, v) writeb(v, (rdev->rmmio) + (reg))
+#define RREG16(reg) readw((rdev->rmmio) + (reg))
+#define WREG16(reg, v) writew(v, (rdev->rmmio) + (reg))
 #define RREG32(reg) r100_mm_rreg(rdev, (reg))
 #define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg)))
 #define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v))
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index b2449629537..df8218bb83a 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -625,7 +625,7 @@ static struct radeon_asic r600_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &r600_cs_parse,
         .copy_blit = &r600_copy_blit,
-        .copy_dma = &r600_copy_blit,
+        .copy_dma = NULL,
         .copy = &r600_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -672,7 +672,7 @@ static struct radeon_asic rs780_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &r600_cs_parse,
         .copy_blit = &r600_copy_blit,
-        .copy_dma = &r600_copy_blit,
+        .copy_dma = NULL,
         .copy = &r600_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -719,7 +719,7 @@ static struct radeon_asic rv770_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &r600_cs_parse,
         .copy_blit = &r600_copy_blit,
-        .copy_dma = &r600_copy_blit,
+        .copy_dma = NULL,
         .copy = &r600_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -766,7 +766,7 @@ static struct radeon_asic evergreen_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &evergreen_cs_parse,
         .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
         .copy = &evergreen_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -813,7 +813,7 @@ static struct radeon_asic sumo_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &evergreen_cs_parse,
         .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
         .copy = &evergreen_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -860,7 +860,7 @@ static struct radeon_asic btc_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &evergreen_cs_parse,
         .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
         .copy = &evergreen_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -907,7 +907,7 @@ static struct radeon_asic cayman_asic = {
         .fence_ring_emit = &r600_fence_ring_emit,
         .cs_parse = &evergreen_cs_parse,
         .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
         .copy = &evergreen_copy_blit,
         .get_engine_clock = &radeon_atom_get_engine_clock,
         .set_engine_clock = &radeon_atom_set_engine_clock,
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index 3d7a0d7c6a9..3dedaa07aac 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -75,7 +75,7 @@ uint32_t r100_pll_rreg(struct radeon_device *rdev, uint32_t reg);
 int r100_copy_blit(struct radeon_device *rdev,
                    uint64_t src_offset,
                    uint64_t dst_offset,
-                   unsigned num_pages,
+                   unsigned num_gpu_pages,
                    struct radeon_fence *fence);
 int r100_set_surface_reg(struct radeon_device *rdev, int reg,
                          uint32_t tiling_flags, uint32_t pitch,
@@ -143,7 +143,7 @@ extern void r100_post_page_flip(struct radeon_device *rdev, int crtc);
 extern int r200_copy_dma(struct radeon_device *rdev,
                          uint64_t src_offset,
                          uint64_t dst_offset,
-                         unsigned num_pages,
+                         unsigned num_gpu_pages,
                          struct radeon_fence *fence);
 void r200_set_safe_registers(struct radeon_device *rdev);
 
@@ -311,7 +311,7 @@ void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
 int r600_ring_test(struct radeon_device *rdev);
 int r600_copy_blit(struct radeon_device *rdev,
                    uint64_t src_offset, uint64_t dst_offset,
-                   unsigned num_pages, struct radeon_fence *fence);
+                   unsigned num_gpu_pages, struct radeon_fence *fence);
 void r600_hpd_init(struct radeon_device *rdev);
 void r600_hpd_fini(struct radeon_device *rdev);
 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd);
@@ -403,7 +403,7 @@ void evergreen_bandwidth_update(struct radeon_device *rdev);
 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
 int evergreen_copy_blit(struct radeon_device *rdev,
                         uint64_t src_offset, uint64_t dst_offset,
-                        unsigned num_pages, struct radeon_fence *fence);
+                        unsigned num_gpu_pages, struct radeon_fence *fence);
 void evergreen_hpd_init(struct radeon_device *rdev);
 void evergreen_hpd_fini(struct radeon_device *rdev);
 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd);
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index bf2b61584cd..a098edcf662 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -85,6 +85,18 @@ static inline struct radeon_i2c_bus_rec radeon_lookup_i2c_gpio(struct radeon_dev
                 for (i = 0; i < num_indices; i++) {
                         gpio = &i2c_info->asGPIO_Info[i];
 
+                        /* r4xx mask is technically not used by the hw, so patch in the legacy mask bits */
+                        if ((rdev->family == CHIP_R420) ||
+                            (rdev->family == CHIP_R423) ||
+                            (rdev->family == CHIP_RV410)) {
+                                if ((le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x0018) ||
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x0019) ||
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x001a)) {
+                                        gpio->ucClkMaskShift = 0x19;
+                                        gpio->ucDataMaskShift = 0x18;
+                                }
+                        }
+
                         /* some evergreen boards have bad data for this entry */
                         if (ASIC_IS_DCE4(rdev)) {
                                 if ((i == 7) &&
@@ -169,6 +181,18 @@ void radeon_atombios_i2c_init(struct radeon_device *rdev)
                         gpio = &i2c_info->asGPIO_Info[i];
                         i2c.valid = false;
 
+                        /* r4xx mask is technically not used by the hw, so patch in the legacy mask bits */
+                        if ((rdev->family == CHIP_R420) ||
+                            (rdev->family == CHIP_R423) ||
+                            (rdev->family == CHIP_RV410)) {
+                                if ((le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x0018) ||
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x0019) ||
+                                    (le16_to_cpu(gpio->usClkMaskRegisterIndex) == 0x001a)) {
+                                        gpio->ucClkMaskShift = 0x19;
+                                        gpio->ucDataMaskShift = 0x18;
+                                }
+                        }
+
                         /* some evergreen boards have bad data for this entry */
                         if (ASIC_IS_DCE4(rdev)) {
                                 if ((i == 7) &&
@@ -2544,7 +2568,11 @@ void radeon_atombios_get_power_modes(struct radeon_device *rdev)
 
         rdev->pm.current_power_state_index = rdev->pm.default_power_state_index;
         rdev->pm.current_clock_mode_index = 0;
-        rdev->pm.current_vddc = rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.voltage;
+        if (rdev->pm.default_power_state_index >= 0)
+                rdev->pm.current_vddc =
+                        rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.voltage;
+        else
+                rdev->pm.current_vddc = 0;
 }
 
 void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable)
diff --git a/drivers/gpu/drm/radeon/radeon_clocks.c b/drivers/gpu/drm/radeon/radeon_clocks.c
index 2d48e7a1474..b6e18c8db9f 100644
--- a/drivers/gpu/drm/radeon/radeon_clocks.c
+++ b/drivers/gpu/drm/radeon/radeon_clocks.c
@@ -96,7 +96,7 @@ uint32_t radeon_legacy_get_memory_clock(struct radeon_device *rdev)
  * Read XTAL (ref clock), SCLK and MCLK from Open Firmware device
  * tree. Hopefully, ATI OF driver is kind enough to fill these
  */
-static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
+static bool radeon_read_clocks_OF(struct drm_device *dev)
 {
         struct radeon_device *rdev = dev->dev_private;
         struct device_node *dp = rdev->pdev->dev.of_node;
@@ -166,7 +166,7 @@ static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
         return true;
 }
 #else
-static bool __devinit radeon_read_clocks_OF(struct drm_device *dev)
+static bool radeon_read_clocks_OF(struct drm_device *dev)
 {
         return false;
 }
@@ -219,6 +219,9 @@ void radeon_get_clock_info(struct drm_device *dev)
                 } else {
                         DRM_INFO("Using generic clock info\n");
 
+                        /* may need to be per card */
+                        rdev->clock.max_pixel_clock = 35000;
+
                         if (rdev->flags & RADEON_IS_IGP) {
                                 p1pll->reference_freq = 1432;
                                 p2pll->reference_freq = 1432;
diff --git a/drivers/gpu/drm/radeon/radeon_combios.c b/drivers/gpu/drm/radeon/radeon_combios.c
index e4594676a07..8bf83c4b414 100644
--- a/drivers/gpu/drm/radeon/radeon_combios.c
+++ b/drivers/gpu/drm/radeon/radeon_combios.c
@@ -620,8 +620,8 @@ static struct radeon_i2c_bus_rec combios_setup_i2c_bus(struct radeon_device *rde
                 i2c.y_data_mask = 0x80;
         } else {
                 /* default masks for ddc pads */
-                i2c.mask_clk_mask = RADEON_GPIO_EN_1;
-                i2c.mask_data_mask = RADEON_GPIO_EN_0;
+                i2c.mask_clk_mask = RADEON_GPIO_MASK_1;
+                i2c.mask_data_mask = RADEON_GPIO_MASK_0;
                 i2c.a_clk_mask = RADEON_GPIO_A_1;
                 i2c.a_data_mask = RADEON_GPIO_A_0;
                 i2c.en_clk_mask = RADEON_GPIO_EN_1;
@@ -779,7 +779,8 @@ void radeon_combios_i2c_init(struct radeon_device *rdev)
                                 }
                         }
                 }
-        } else if (rdev->family >= CHIP_R200) {
+        } else if ((rdev->family == CHIP_R200) ||
+                   (rdev->family >= CHIP_R300)) {
                 /* 0x68 */
                 i2c = combios_setup_i2c_bus(rdev, DDC_MONID, 0, 0);
                 rdev->i2c_bus[3] = radeon_i2c_create(dev, &i2c, "MONID");
@@ -2556,6 +2557,7 @@ void radeon_combios_get_power_modes(struct radeon_device *rdev)
         u16 offset, misc, misc2 = 0;
         u8 rev, blocks, tmp;
         int state_index = 0;
+        struct radeon_i2c_bus_rec i2c_bus;
 
         rdev->pm.default_power_state_index = -1;
 
@@ -2574,7 +2576,6 @@ void radeon_combios_get_power_modes(struct radeon_device *rdev)
         offset = combios_get_table_offset(dev, COMBIOS_OVERDRIVE_INFO_TABLE);
         if (offset) {
                 u8 thermal_controller = 0, gpio = 0, i2c_addr = 0, clk_bit = 0, data_bit = 0;
-                struct radeon_i2c_bus_rec i2c_bus;
 
                 rev = RBIOS8(offset);
 
@@ -2616,6 +2617,25 @@ void radeon_combios_get_power_modes(struct radeon_device *rdev)
                                 i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
                         }
                 }
+        } else {
+                /* boards with a thermal chip, but no overdrive table */
+
+                /* Asus 9600xt has an f75375 on the monid bus */
+                if ((dev->pdev->device == 0x4152) &&
+                    (dev->pdev->subsystem_vendor == 0x1043) &&
+                    (dev->pdev->subsystem_device == 0xc002)) {
+                        i2c_bus = combios_setup_i2c_bus(rdev, DDC_MONID, 0, 0);
+                        rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
+                        if (rdev->pm.i2c_bus) {
+                                struct i2c_board_info info = { };
+                                const char *name = "f75375";
+                                info.addr = 0x28;
+                                strlcpy(info.type, name, sizeof(info.type));
+                                i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
+                                DRM_INFO("Possible %s thermal controller at 0x%02x\n",
+                                         name, info.addr);
+                        }
+                }
         }
 
         if (rdev->flags & RADEON_IS_MOBILITY) {
@@ -3278,6 +3298,14 @@ void radeon_combios_asic_init(struct drm_device *dev)
             rdev->pdev->subsystem_device == 0x30a4)
                 return;
 
+        /* quirk for rs4xx Compaq Presario V5245EU laptop to make it resume
+         * - it hangs on resume inside the dynclk 1 table.
+         */
+        if (rdev->family == CHIP_RS480 &&
+            rdev->pdev->subsystem_vendor == 0x103c &&
+            rdev->pdev->subsystem_device == 0x30ae)
+                return;
+
         /* DYN CLK 1 */
         table = combios_get_table_offset(dev, COMBIOS_DYN_CLK_1_TABLE);
         if (table)
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 9792d4ffdc8..a58b4522404 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -44,8 +44,6 @@ extern void
 radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
                              struct drm_connector *drm_connector);
 
-bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector);
-
 void radeon_connector_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
@@ -60,18 +58,20 @@ void radeon_connector_hotplug(struct drm_connector *connector)
 
         radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
 
-        /* powering up/down the eDP panel generates hpd events which
-         * can interfere with modesetting.
-         */
-        if (connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+        /* if the connector is already off, don't turn it back on */
+        if (connector->dpms != DRM_MODE_DPMS_ON)
                 return;
 
-        /* pre-r600 did not always have the hpd pins mapped accurately to connectors */
-        if (rdev->family >= CHIP_R600) {
-                if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd))
-                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
-                else
+        /* just deal with DP (not eDP) here. */
+        if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
+                int saved_dpms = connector->dpms;
+
+                /* Only turn off the display it it's physically disconnected */
+                if (!radeon_hpd_sense(rdev, radeon_connector->hpd.hpd))
                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+                else if (radeon_dp_needs_link_train(radeon_connector))
+                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+                connector->dpms = saved_dpms;
         }
 }
 
@@ -430,16 +430,73 @@ int radeon_connector_set_property(struct drm_connector *connector, struct drm_pr
         return 0;
 }
 
+/*
+ * Some integrated ATI Radeon chipset implementations (e. g.
+ * Asus M2A-VM HDMI) may indicate the availability of a DDC,
+ * even when there's no monitor connected. For these connectors
+ * following DDC probe extension will be applied: check also for the
+ * availability of EDID with at least a correct EDID header. Only then,
+ * DDC is assumed to be available. This prevents drm_get_edid() and
+ * drm_edid_block_valid() from periodically dumping data and kernel
+ * errors into the logs and onto the terminal.
+ */
+static bool radeon_connector_needs_extended_probe(struct radeon_device *dev,
+                                     uint32_t supported_device,
+                                     int connector_type)
+{
+        /* Asus M2A-VM HDMI board sends data to i2c bus even,
+         * if HDMI add-on card is not plugged in or HDMI is disabled in
+         * BIOS. Valid DDC can only be assumed, if also a valid EDID header
+         * can be retrieved via i2c bus during DDC probe */
+        if ((dev->pdev->device == 0x791e) &&
+            (dev->pdev->subsystem_vendor == 0x1043) &&
+            (dev->pdev->subsystem_device == 0x826d)) {
+                if ((connector_type == DRM_MODE_CONNECTOR_HDMIA) &&
+                    (supported_device == ATOM_DEVICE_DFP2_SUPPORT))
+                        return true;
+        }
+        /* ECS A740GM-M with ATI RADEON 2100 sends data to i2c bus
+         * for a DVI connector that is not implemented */
+        if ((dev->pdev->device == 0x796e) &&
+            (dev->pdev->subsystem_vendor == 0x1019) &&
+            (dev->pdev->subsystem_device == 0x2615)) {
+                if ((connector_type == DRM_MODE_CONNECTOR_DVID) &&
+                    (supported_device == ATOM_DEVICE_DFP2_SUPPORT))
+                        return true;
+        }
+        /* TOSHIBA Satellite L300D with ATI Mobility Radeon x1100
+         * (RS690M) sends data to i2c bus for a HDMI connector that
+         * is not implemented */
+        if ((dev->pdev->device == 0x791f) &&
+            (dev->pdev->subsystem_vendor == 0x1179) &&
+            (dev->pdev->subsystem_device == 0xff68)) {
+                if ((connector_type == DRM_MODE_CONNECTOR_HDMIA) &&
+                    (supported_device == ATOM_DEVICE_DFP2_SUPPORT))
+                        return true;
+        }
+
+        /* Default: no EDID header probe required for DDC probing */
+        return false;
+}
+
 static void radeon_fixup_lvds_native_mode(struct drm_encoder *encoder,
                                           struct drm_connector *connector)
 {
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
         struct drm_display_mode *native_mode = &radeon_encoder->native_mode;
+        struct drm_display_mode *t, *mode;
+
+        /* If the EDID preferred mode doesn't match the native mode, use it */
+        list_for_each_entry_safe(mode, t, &connector->probed_modes, head) {
+                if (mode->type & DRM_MODE_TYPE_PREFERRED) {
+                        if (mode->hdisplay != native_mode->hdisplay ||
+                            mode->vdisplay != native_mode->vdisplay)
+                                memcpy(native_mode, mode, sizeof(*mode));
+                }
+        }
 
         /* Try to get native mode details from EDID if necessary */
         if (!native_mode->clock) {
-                struct drm_display_mode *t, *mode;
-
                 list_for_each_entry_safe(mode, t, &connector->probed_modes, head) {
                         if (mode->hdisplay == native_mode->hdisplay &&
                             mode->vdisplay == native_mode->vdisplay) {
@@ -450,6 +507,7 @@ static void radeon_fixup_lvds_native_mode(struct drm_encoder *encoder,
                         }
                 }
         }
+
         if (!native_mode->clock) {
                 DRM_DEBUG_KMS("No LVDS native mode details, disabling RMX\n");
                 radeon_encoder->rmx_type = RMX_OFF;
@@ -661,8 +719,10 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
                 ret = connector_status_disconnected;
 
         if (radeon_connector->ddc_bus)
-                dret = radeon_ddc_probe(radeon_connector);
+                dret = radeon_ddc_probe(radeon_connector,
+                                        radeon_connector->requires_extended_probe);
         if (dret) {
+                radeon_connector->detected_by_load = false;
                 if (radeon_connector->edid) {
                         kfree(radeon_connector->edid);
                         radeon_connector->edid = NULL;
@@ -689,12 +749,21 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
         } else {
 
                 /* if we aren't forcing don't do destructive polling */
-                if (!force)
-                        return connector->status;
+                if (!force) {
+                        /* only return the previous status if we last
+                         * detected a monitor via load.
+                         */
+                        if (radeon_connector->detected_by_load)
+                                return connector->status;
+                        else
+                                return ret;
+                }
 
                 if (radeon_connector->dac_load_detect && encoder) {
                         encoder_funcs = encoder->helper_private;
                         ret = encoder_funcs->detect(encoder, connector);
+                        if (ret != connector_status_disconnected)
+                                radeon_connector->detected_by_load = true;
                 }
         }
 
@@ -833,8 +902,10 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
         bool dret = false;
 
         if (radeon_connector->ddc_bus)
-                dret = radeon_ddc_probe(radeon_connector);
+                dret = radeon_ddc_probe(radeon_connector,
+                                        radeon_connector->requires_extended_probe);
         if (dret) {
+                radeon_connector->detected_by_load = false;
                 if (radeon_connector->edid) {
                         kfree(radeon_connector->edid);
                         radeon_connector->edid = NULL;
@@ -897,8 +968,18 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
         if ((ret == connector_status_connected) && (radeon_connector->use_digital == true))
                 goto out;
 
+        /* DVI-D and HDMI-A are digital only */
+        if ((connector->connector_type == DRM_MODE_CONNECTOR_DVID) ||
+            (connector->connector_type == DRM_MODE_CONNECTOR_HDMIA))
+                goto out;
+
+        /* if we aren't forcing don't do destructive polling */
         if (!force) {
-                ret = connector->status;
+                /* only return the previous status if we last
+                 * detected a monitor via load.
+                 */
+                if (radeon_connector->detected_by_load)
+                        ret = connector->status;
                 goto out;
         }
 
@@ -923,6 +1004,8 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
                                         if (ret == connector_status_connected) {
                                                 radeon_connector->use_digital = false;
                                         }
+                                        if (ret != connector_status_disconnected)
+                                                radeon_connector->detected_by_load = true;
                                 }
                                 break;
                         }
@@ -1119,7 +1202,8 @@ static int radeon_dp_get_modes(struct drm_connector *connector)
                 }
         } else {
                 /* need to setup ddc on the bridge */
-                if (radeon_connector_encoder_is_dp_bridge(connector)) {
+                if (radeon_connector_encoder_get_dp_bridge_encoder_id(connector) !=
+                        ENCODER_OBJECT_ID_NONE) {
                         if (encoder)
                                 radeon_atom_ext_encoder_setup_ddc(encoder);
                 }
@@ -1129,13 +1213,12 @@ static int radeon_dp_get_modes(struct drm_connector *connector)
         return ret;
 }
 
-bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector)
+u16 radeon_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *connector)
 {
         struct drm_mode_object *obj;
         struct drm_encoder *encoder;
         struct radeon_encoder *radeon_encoder;
         int i;
-        bool found = false;
 
         for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
                 if (connector->encoder_ids[i] == 0)
@@ -1151,14 +1234,13 @@ bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector)
                 switch (radeon_encoder->encoder_id) {
                 case ENCODER_OBJECT_ID_TRAVIS:
                 case ENCODER_OBJECT_ID_NUTMEG:
-                        found = true;
-                        break;
+                        return radeon_encoder->encoder_id;
                 default:
                         break;
                 }
         }
 
-        return found;
+        return ENCODER_OBJECT_ID_NONE;
 }
 
 bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector)
@@ -1235,12 +1317,25 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
                 if (!radeon_dig_connector->edp_on)
                         atombios_set_edp_panel_power(connector,
                                                      ATOM_TRANSMITTER_ACTION_POWER_OFF);
-        } else {
-                /* need to setup ddc on the bridge */
-                if (radeon_connector_encoder_is_dp_bridge(connector)) {
-                        if (encoder)
-                                radeon_atom_ext_encoder_setup_ddc(encoder);
+        } else if (radeon_connector_encoder_get_dp_bridge_encoder_id(connector) !=
+                   ENCODER_OBJECT_ID_NONE) {
+                /* DP bridges are always DP */
+                radeon_dig_connector->dp_sink_type = CONNECTOR_OBJECT_ID_DISPLAYPORT;
+                /* get the DPCD from the bridge */
+                radeon_dp_getdpcd(radeon_connector);
+
+                if (encoder) {
+                        /* setup ddc on the bridge */
+                        radeon_atom_ext_encoder_setup_ddc(encoder);
+                        if (radeon_ddc_probe(radeon_connector,
+                                             radeon_connector->requires_extended_probe)) /* try DDC */
+                                ret = connector_status_connected;
+                        else if (radeon_connector->dac_load_detect) { /* try load detection */
+                                struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
+                                ret = encoder_funcs->detect(encoder, connector);
+                        }
                 }
+        } else {
                 radeon_dig_connector->dp_sink_type = radeon_dp_getsinktype(radeon_connector);
                 if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) {
                         ret = connector_status_connected;
@@ -1251,20 +1346,11 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
                                 if (radeon_dp_getdpcd(radeon_connector))
                                         ret = connector_status_connected;
                         } else {
-                                if (radeon_ddc_probe(radeon_connector))
+                                if (radeon_ddc_probe(radeon_connector,
+                                                     radeon_connector->requires_extended_probe))
                                         ret = connector_status_connected;
                         }
                 }
-
-                if ((ret == connector_status_disconnected) &&
-                    radeon_connector->dac_load_detect) {
-                        struct drm_encoder *encoder = radeon_best_single_encoder(connector);
-                        struct drm_encoder_helper_funcs *encoder_funcs;
-                        if (encoder) {
-                                encoder_funcs = encoder->helper_private;
-                                ret = encoder_funcs->detect(encoder, connector);
-                        }
-                }
         }
 
         radeon_connector_update_scratch_regs(connector, ret);
@@ -1406,6 +1492,9 @@ radeon_add_atom_connector(struct drm_device *dev,
         radeon_connector->shared_ddc = shared_ddc;
         radeon_connector->connector_object_id = connector_object_id;
         radeon_connector->hpd = *hpd;
+        radeon_connector->requires_extended_probe =
+                radeon_connector_needs_extended_probe(rdev, supported_device,
+                                                        connector_type);
         radeon_connector->router = *router;
         if (router->ddc_valid || router->cd_valid) {
                 radeon_connector->router_bus = radeon_i2c_lookup(rdev, &router->i2c_info);
@@ -1752,6 +1841,9 @@ radeon_add_legacy_connector(struct drm_device *dev,
         radeon_connector->devices = supported_device;
         radeon_connector->connector_object_id = connector_object_id;
         radeon_connector->hpd = *hpd;
+        radeon_connector->requires_extended_probe =
+                radeon_connector_needs_extended_probe(rdev, supported_device,
+                                                        connector_type);
         switch (connector_type) {
         case DRM_MODE_CONNECTOR_VGA:
                 drm_connector_init(dev, &radeon_connector->base, &radeon_vga_connector_funcs, connector_type);
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index 75867792a4e..045ec59478f 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -2115,7 +2115,7 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
 
         if (drm_pci_device_is_agp(dev))
                 dev_priv->flags |= RADEON_IS_AGP;
-        else if (drm_pci_device_is_pcie(dev))
+        else if (pci_is_pcie(dev->pdev))
                 dev_priv->flags |= RADEON_IS_PCIE;
         else
                 dev_priv->flags |= RADEON_IS_PCI;
diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c
index 3189a7efb2e..fde25c0d65a 100644
--- a/drivers/gpu/drm/radeon/radeon_cursor.c
+++ b/drivers/gpu/drm/radeon/radeon_cursor.c
@@ -208,23 +208,25 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc,
         int xorigin = 0, yorigin = 0;
         int w = radeon_crtc->cursor_width;
 
-        if (x < 0)
-                xorigin = -x + 1;
-        if (y < 0)
-                yorigin = -y + 1;
-        if (xorigin >= CURSOR_WIDTH)
-                xorigin = CURSOR_WIDTH - 1;
-        if (yorigin >= CURSOR_HEIGHT)
-                yorigin = CURSOR_HEIGHT - 1;
-
         if (ASIC_IS_AVIVO(rdev)) {
-                int i = 0;
-                struct drm_crtc *crtc_p;
-
                 /* avivo cursor are offset into the total surface */
                 x += crtc->x;
                 y += crtc->y;
-                DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
+        }
+        DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
+
+        if (x < 0) {
+                xorigin = min(-x, CURSOR_WIDTH - 1);
+                x = 0;
+        }
+        if (y < 0) {
+                yorigin = min(-y, CURSOR_HEIGHT - 1);
+                y = 0;
+        }
+
+        if (ASIC_IS_AVIVO(rdev)) {
+                int i = 0;
+                struct drm_crtc *crtc_p;
 
                 /* avivo cursor image can't end on 128 pixel boundary or
                  * go past the end of the frame if both crtcs are enabled
@@ -253,16 +255,12 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc,
 
         radeon_lock_cursor(crtc, true);
         if (ASIC_IS_DCE4(rdev)) {
-                WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset,
-                       ((xorigin ? 0 : x) << 16) |
-                       (yorigin ? 0 : y));
+                WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y);
                 WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin);
                 WREG32(EVERGREEN_CUR_SIZE + radeon_crtc->crtc_offset,
                        ((w - 1) << 16) | (radeon_crtc->cursor_height - 1));
         } else if (ASIC_IS_AVIVO(rdev)) {
-                WREG32(AVIVO_D1CUR_POSITION + radeon_crtc->crtc_offset,
-                             ((xorigin ? 0 : x) << 16) |
-                             (yorigin ? 0 : y));
+                WREG32(AVIVO_D1CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y);
                 WREG32(AVIVO_D1CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin);
                 WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset,
                        ((w - 1) << 16) | (radeon_crtc->cursor_height - 1));
@@ -276,8 +274,8 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc,
                         | yorigin));
                 WREG32(RADEON_CUR_HORZ_VERT_POSN + radeon_crtc->crtc_offset,
                        (RADEON_CUR_LOCK
-                        | ((xorigin ? 0 : x) << 16)
-                        | (yorigin ? 0 : y)));
+                        | (x << 16)
+                        | y));
                 /* offset is from DISP(2)_BASE_ADDRESS */
                 WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, (radeon_crtc->legacy_cursor_offset +
                                                                       (yorigin * 256)));
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 7cfaa7e2f3b..50d105a1c74 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -32,6 +32,7 @@
 #include <drm/radeon_drm.h>
 #include <linux/vgaarb.h>
 #include <linux/vga_switcheroo.h>
+#include <linux/efi.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "atom.h"
@@ -223,8 +224,11 @@ int radeon_wb_init(struct radeon_device *rdev)
         if (radeon_no_wb == 1)
                 rdev->wb.enabled = false;
         else {
-                /* often unreliable on AGP */
                 if (rdev->flags & RADEON_IS_AGP) {
+                        /* often unreliable on AGP */
+                        rdev->wb.enabled = false;
+                } else if (rdev->family < CHIP_R300) {
+                        /* often unreliable on pre-r300 */
                         rdev->wb.enabled = false;
                 } else {
                         rdev->wb.enabled = true;
@@ -300,6 +304,8 @@ void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64
                 mc->mc_vram_size = mc->aper_size;
         }
         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
+        if (radeon_vram_limit && radeon_vram_limit < mc->real_vram_size)
+                mc->real_vram_size = radeon_vram_limit;
         dev_info(rdev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
                         mc->mc_vram_size >> 20, mc->vram_start,
                         mc->vram_end, mc->real_vram_size >> 20);
@@ -348,6 +354,9 @@ bool radeon_card_posted(struct radeon_device *rdev)
 {
         uint32_t reg;
 
+        if (efi_enabled && rdev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE)
+                return false;
+
         /* first check CRTCs */
         if (ASIC_IS_DCE41(rdev)) {
                 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
@@ -704,8 +713,9 @@ int radeon_device_init(struct radeon_device *rdev,
         rdev->gpu_lockup = false;
         rdev->accel_working = false;
 
-        DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X).\n",
-                radeon_family_name[rdev->family], pdev->vendor, pdev->device);
+        DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n",
+                radeon_family_name[rdev->family], pdev->vendor, pdev->device,
+                pdev->subsystem_vendor, pdev->subsystem_device);
 
         /* mutex initialization are all done here so we
          * can recall function without having locking issues */
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 292f73f0ddb..07ac48162a1 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -282,7 +282,7 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
         spin_lock_irqsave(&rdev->ddev->event_lock, flags);
         work = radeon_crtc->unpin_work;
         if (work == NULL ||
-            !radeon_fence_signaled(work->fence)) {
+            (work->fence && !radeon_fence_signaled(work->fence))) {
                 spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
                 return;
         }
@@ -348,7 +348,6 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         struct radeon_framebuffer *new_radeon_fb;
         struct drm_gem_object *obj;
         struct radeon_bo *rbo;
-        struct radeon_fence *fence;
         struct radeon_unpin_work *work;
         unsigned long flags;
         u32 tiling_flags, pitch_pixels;
@@ -359,16 +358,9 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         if (work == NULL)
                 return -ENOMEM;
 
-        r = radeon_fence_create(rdev, &fence);
-        if (unlikely(r != 0)) {
-                kfree(work);
-                DRM_ERROR("flip queue: failed to create fence.\n");
-                return -ENOMEM;
-        }
         work->event = event;
         work->rdev = rdev;
         work->crtc_id = radeon_crtc->crtc_id;
-        work->fence = radeon_fence_ref(fence);
         old_radeon_fb = to_radeon_framebuffer(crtc->fb);
         new_radeon_fb = to_radeon_framebuffer(fb);
         /* schedule unpin of the old buffer */
@@ -377,6 +369,10 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         drm_gem_object_reference(obj);
         rbo = gem_to_radeon_bo(obj);
         work->old_rbo = rbo;
+        obj = new_radeon_fb->obj;
+        rbo = gem_to_radeon_bo(obj);
+        if (rbo->tbo.sync_obj)
+                work->fence = radeon_fence_ref(rbo->tbo.sync_obj);
         INIT_WORK(&work->work, radeon_unpin_work_func);
 
         /* We borrow the event spin lock for protecting unpin_work */
@@ -391,9 +387,6 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
         /* pin the new buffer */
-        obj = new_radeon_fb->obj;
-        rbo = gem_to_radeon_bo(obj);
-
         DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",
                          work->old_rbo, rbo);
 
@@ -461,37 +454,18 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
                 goto pflip_cleanup1;
         }
 
-        /* 32 ought to cover us */
-        r = radeon_ring_lock(rdev, 32);
-        if (r) {
-                DRM_ERROR("failed to lock the ring before flip\n");
-                goto pflip_cleanup2;
-        }
-
-        /* emit the fence */
-        radeon_fence_emit(rdev, fence);
         /* set the proper interrupt */
         radeon_pre_page_flip(rdev, radeon_crtc->crtc_id);
-        /* fire the ring */
-        radeon_ring_unlock_commit(rdev);
 
         return 0;
 
-pflip_cleanup2:
-        drm_vblank_put(dev, radeon_crtc->crtc_id);
-
 pflip_cleanup1:
-        r = radeon_bo_reserve(rbo, false);
-        if (unlikely(r != 0)) {
+        if (unlikely(radeon_bo_reserve(rbo, false) != 0)) {
                 DRM_ERROR("failed to reserve new rbo in error path\n");
                 goto pflip_cleanup;
         }
-        r = radeon_bo_unpin(rbo);
-        if (unlikely(r != 0)) {
-                radeon_bo_unreserve(rbo);
-                r = -EINVAL;
+        if (unlikely(radeon_bo_unpin(rbo) != 0)) {
                 DRM_ERROR("failed to unpin new rbo in error path\n");
-                goto pflip_cleanup;
         }
         radeon_bo_unreserve(rbo);
 
@@ -499,9 +473,9 @@ pflip_cleanup:
         spin_lock_irqsave(&dev->event_lock, flags);
         radeon_crtc->unpin_work = NULL;
 unlock_free:
-        drm_gem_object_unreference_unlocked(old_radeon_fb->obj);
         spin_unlock_irqrestore(&dev->event_lock, flags);
-        radeon_fence_unref(&fence);
+        drm_gem_object_unreference_unlocked(old_radeon_fb->obj);
+        radeon_fence_unref(&work->fence);
         kfree(work);
 
         return r;
@@ -733,16 +707,22 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
                 radeon_router_select_ddc_port(radeon_connector);
 
         if ((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
-            (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)) {
+            (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) ||
+            (radeon_connector_encoder_get_dp_bridge_encoder_id(&radeon_connector->base) !=
+             ENCODER_OBJECT_ID_NONE)) {
                 struct radeon_connector_atom_dig *dig = radeon_connector->con_priv;
+
                 if ((dig->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT ||
                      dig->dp_sink_type == CONNECTOR_OBJECT_ID_eDP) && dig->dp_i2c_bus)
-                        radeon_connector->edid = drm_get_edid(&radeon_connector->base, &dig->dp_i2c_bus->adapter);
-        }
-        if (!radeon_connector->ddc_bus)
-                return -1;
-        if (!radeon_connector->edid) {
-                radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter);
+                        radeon_connector->edid = drm_get_edid(&radeon_connector->base,
+                                                              &dig->dp_i2c_bus->adapter);
+                else if (radeon_connector->ddc_bus && !radeon_connector->edid)
+                        radeon_connector->edid = drm_get_edid(&radeon_connector->base,
+                                                              &radeon_connector->ddc_bus->adapter);
+        } else {
+                if (radeon_connector->ddc_bus && !radeon_connector->edid)
+                        radeon_connector->edid = drm_get_edid(&radeon_connector->base,
+                                                              &radeon_connector->ddc_bus->adapter);
         }
 
         if (!radeon_connector->edid) {
@@ -777,8 +757,17 @@ static int radeon_ddc_dump(struct drm_connector *connector)
         if (!radeon_connector->ddc_bus)
                 return -1;
         edid = drm_get_edid(connector, &radeon_connector->ddc_bus->adapter);
+        /* Log EDID retrieval status here. In particular with regard to
+         * connectors with requires_extended_probe flag set, that will prevent
+         * function radeon_dvi_detect() to fetch EDID on this connector,
+         * as long as there is no valid EDID header found */
         if (edid) {
+                DRM_INFO("Radeon display connector %s: Found valid EDID",
+                                drm_get_connector_name(connector));
                 kfree(edid);
+        } else {
+                DRM_INFO("Radeon display connector %s: No monitor connected or invalid EDID",
+                                drm_get_connector_name(connector));
         }
         return ret;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 73dfbe8e5f9..c12b0775d68 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -51,9 +51,10 @@
  *   2.8.0 - pageflip support, r500 US_FORMAT regs. r500 ARGB2101010 colorbuf, r300->r500 CMASK, clock crystal query
  *   2.9.0 - r600 tiling (s3tc,rgtc) working, SET_PREDICATION packet 3 on r600 + eg, backend query
  *   2.10.0 - fusion 2D tiling
+ *   2.11.0 - backend map, initial compute support for the CS checker
  */
 #define KMS_DRIVER_MAJOR        2
-#define KMS_DRIVER_MINOR        10
+#define KMS_DRIVER_MINOR        11
 #define KMS_DRIVER_PATCHLEVEL   0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
@@ -117,6 +118,7 @@ int radeon_audio = 0;
 int radeon_disp_priority = 0;
 int radeon_hw_i2c = 0;
 int radeon_pcie_gen2 = 0;
+int radeon_msi = -1;
 
 MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers");
 module_param_named(no_wb, radeon_no_wb, int, 0444);
@@ -163,6 +165,9 @@ module_param_named(hw_i2c, radeon_hw_i2c, int, 0444);
 MODULE_PARM_DESC(pcie_gen2, "PCIE Gen2 mode (1 = enable)");
 module_param_named(pcie_gen2, radeon_pcie_gen2, int, 0444);
 
+MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
+module_param_named(msi, radeon_msi, int, 0444);
+
 static int radeon_suspend(struct drm_device *dev, pm_message_t state)
 {
         drm_radeon_private_t *dev_priv = dev->dev_private;
diff --git a/drivers/gpu/drm/radeon/radeon_encoders.c b/drivers/gpu/drm/radeon/radeon_encoders.c
index b293487e5aa..9838865e223 100644
--- a/drivers/gpu/drm/radeon/radeon_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_encoders.c
@@ -266,7 +266,7 @@ struct drm_encoder *radeon_atom_get_external_encoder(struct drm_encoder *encoder
         return NULL;
 }
 
-bool radeon_encoder_is_dp_bridge(struct drm_encoder *encoder)
+u16 radeon_encoder_get_dp_bridge_encoder_id(struct drm_encoder *encoder)
 {
         struct drm_encoder *other_encoder = radeon_atom_get_external_encoder(encoder);
 
@@ -368,7 +368,7 @@ static bool radeon_atom_mode_fixup(struct drm_encoder *encoder,
 
         if (ASIC_IS_DCE3(rdev) &&
             ((radeon_encoder->active_device & (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) ||
-             radeon_encoder_is_dp_bridge(encoder))) {
+             (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE))) {
                 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
                 radeon_dp_set_link_config(connector, mode);
         }
@@ -658,7 +658,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
         struct radeon_connector_atom_dig *dig_connector;
 
         /* dp bridges are always DP */
-        if (radeon_encoder_is_dp_bridge(encoder))
+        if (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)
                 return ATOM_ENCODER_MODE_DP;
 
         /* DVO is always DVO */
@@ -1507,7 +1507,14 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 switch (mode) {
                 case DRM_MODE_DPMS_ON:
                         args.ucAction = ATOM_ENABLE;
-                        atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+                        /* workaround for DVOOutputControl on some RS690 systems */
+                        if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_DDI) {
+                                u32 reg = RREG32(RADEON_BIOS_3_SCRATCH);
+                                WREG32(RADEON_BIOS_3_SCRATCH, reg & ~ATOM_S3_DFP2I_ACTIVE);
+                                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+                                WREG32(RADEON_BIOS_3_SCRATCH, reg);
+                        } else
+                                atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
                         if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
                                 args.ucAction = ATOM_LCD_BLON;
                                 atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
@@ -1631,7 +1638,17 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder)
                         break;
                 case 2:
                         args.v2.ucCRTC = radeon_crtc->crtc_id;
-                        args.v2.ucEncodeMode = atombios_get_encoder_mode(encoder);
+                        if (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE) {
+                                struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+
+                                if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)
+                                        args.v2.ucEncodeMode = ATOM_ENCODER_MODE_LVDS;
+                                else if (connector->connector_type == DRM_MODE_CONNECTOR_VGA)
+                                        args.v2.ucEncodeMode = ATOM_ENCODER_MODE_CRT;
+                                else
+                                        args.v2.ucEncodeMode = atombios_get_encoder_mode(encoder);
+                        } else
+                                args.v2.ucEncodeMode = atombios_get_encoder_mode(encoder);
                         switch (radeon_encoder->encoder_id) {
                         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
                         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
@@ -1748,9 +1765,17 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder)
         /* DCE4/5 */
         if (ASIC_IS_DCE4(rdev)) {
                 dig = radeon_encoder->enc_priv;
-                if (ASIC_IS_DCE41(rdev))
-                        return radeon_crtc->crtc_id;
-                else {
+                if (ASIC_IS_DCE41(rdev)) {
+                        /* ontario follows DCE4 */
+                        if (rdev->family == CHIP_PALM) {
+                                if (dig->linkb)
+                                        return 1;
+                                else
+                                        return 0;
+                        } else
+                                /* llano follows DCE3.2 */
+                                return radeon_crtc->crtc_id;
+                } else {
                         switch (radeon_encoder->encoder_id) {
                         case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
                                 if (dig->linkb)
@@ -2074,7 +2099,8 @@ static void radeon_atom_encoder_prepare(struct drm_encoder *encoder)
 
         if ((radeon_encoder->active_device &
              (ATOM_DEVICE_DFP_SUPPORT | ATOM_DEVICE_LCD_SUPPORT)) ||
-            radeon_encoder_is_dp_bridge(encoder)) {
+            (radeon_encoder_get_dp_bridge_encoder_id(encoder) !=
+             ENCODER_OBJECT_ID_NONE)) {
                 struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
                 if (dig)
                         dig->dig_encoder = radeon_atom_pick_dig_encoder(encoder);
@@ -2323,6 +2349,9 @@ radeon_add_atom_encoder(struct drm_device *dev,
         default:
                 encoder->possible_crtcs = 0x3;
                 break;
+        case 4:
+                encoder->possible_crtcs = 0xf;
+                break;
         case 6:
                 encoder->possible_crtcs = 0x3f;
                 break;
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 021d2b6b556..7fd4e3e5ad5 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -29,7 +29,7 @@
  *    Dave Airlie
  */
 #include <linux/seq_file.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 #include <linux/wait.h>
 #include <linux/list.h>
 #include <linux/kref.h>
diff --git a/drivers/gpu/drm/radeon/radeon_i2c.c b/drivers/gpu/drm/radeon/radeon_i2c.c
index 781196db792..6c111c1fa3f 100644
--- a/drivers/gpu/drm/radeon/radeon_i2c.c
+++ b/drivers/gpu/drm/radeon/radeon_i2c.c
@@ -32,17 +32,17 @@
  * radeon_ddc_probe
  *
  */
-bool radeon_ddc_probe(struct radeon_connector *radeon_connector)
+bool radeon_ddc_probe(struct radeon_connector *radeon_connector, bool requires_extended_probe)
 {
-        u8 out_buf[] = { 0x0, 0x0};
-        u8 buf[2];
+        u8 out = 0x0;
+        u8 buf[8];
         int ret;
         struct i2c_msg msgs[] = {
                 {
                         .addr = 0x50,
                         .flags = 0,
                         .len = 1,
-                        .buf = out_buf,
+                        .buf = &out,
                 },
                 {
                         .addr = 0x50,
@@ -52,15 +52,31 @@ bool radeon_ddc_probe(struct radeon_connector *radeon_connector)
                 }
         };
 
+        /* Read 8 bytes from i2c for extended probe of EDID header */
+        if (requires_extended_probe)
+                msgs[1].len = 8;
+
         /* on hw with routers, select right port */
         if (radeon_connector->router.ddc_valid)
                 radeon_router_select_ddc_port(radeon_connector);
 
         ret = i2c_transfer(&radeon_connector->ddc_bus->adapter, msgs, 2);
-        if (ret == 2)
-                return true;
-
-        return false;
+        if (ret != 2)
+                /* Couldn't find an accessible DDC on this connector */
+                return false;
+        if (requires_extended_probe) {
+                /* Probe also for valid EDID header
+                 * EDID header starts with:
+                 * 0x00,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0x00.
+                 * Only the first 6 bytes must be valid as
+                 * drm_edid_block_valid() can fix the last 2 bytes */
+                if (drm_edid_header_is_valid(buf) < 6) {
+                        /* Couldn't find an accessible EDID on this
+                         * connector */
+                        return false;
+                }
+        }
+        return true;
 }
 
 /* bit banging i2c */
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 9ec830c77af..5feb6e9edd8 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -108,6 +108,52 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
         radeon_irq_set(rdev);
 }
 
+static bool radeon_msi_ok(struct radeon_device *rdev)
+{
+        /* RV370/RV380 was first asic with MSI support */
+        if (rdev->family < CHIP_RV380)
+                return false;
+
+        /* MSIs don't work on AGP */
+        if (rdev->flags & RADEON_IS_AGP)
+                return false;
+
+        /* force MSI on */
+        if (radeon_msi == 1)
+                return true;
+        else if (radeon_msi == 0)
+                return false;
+
+        /* Quirks */
+        /* HP RS690 only seems to work with MSIs. */
+        if ((rdev->pdev->device == 0x791f) &&
+            (rdev->pdev->subsystem_vendor == 0x103c) &&
+            (rdev->pdev->subsystem_device == 0x30c2))
+                return true;
+
+        /* Dell RS690 only seems to work with MSIs. */
+        if ((rdev->pdev->device == 0x791f) &&
+            (rdev->pdev->subsystem_vendor == 0x1028) &&
+            (rdev->pdev->subsystem_device == 0x01fc))
+                return true;
+
+        /* Dell RS690 only seems to work with MSIs. */
+        if ((rdev->pdev->device == 0x791f) &&
+            (rdev->pdev->subsystem_vendor == 0x1028) &&
+            (rdev->pdev->subsystem_device == 0x01fd))
+                return true;
+
+        if (rdev->flags & RADEON_IS_IGP) {
+                /* APUs work fine with MSIs */
+                if (rdev->family >= CHIP_PALM)
+                        return true;
+                /* lots of IGPs have problems with MSIs */
+                return false;
+        }
+
+        return true;
+}
+
 int radeon_irq_kms_init(struct radeon_device *rdev)
 {
         int i;
@@ -124,12 +170,8 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
         }
         /* enable msi */
         rdev->msi_enabled = 0;
-        /* MSIs don't seem to work reliably on all IGP
-         * chips.  Disable MSI on them for now.
-         */
-        if ((rdev->family >= CHIP_RV380) &&
-            ((!(rdev->flags & RADEON_IS_IGP)) || (rdev->family >= CHIP_PALM)) &&
-            (!(rdev->flags & RADEON_IS_AGP))) {
+
+        if (radeon_msi_ok(rdev)) {
                 int ret = pci_enable_msi(rdev->pdev);
                 if (!ret) {
                         rdev->msi_enabled = 1;
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index bd58af65858..be2c1224e68 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -60,7 +60,7 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
         /* update BUS flag */
         if (drm_pci_device_is_agp(dev)) {
                 flags |= RADEON_IS_AGP;
-        } else if (drm_pci_device_is_pcie(dev)) {
+        } else if (pci_is_pcie(dev->pdev)) {
                 flags |= RADEON_IS_PCIE;
         } else {
                 flags |= RADEON_IS_PCI;
@@ -237,6 +237,19 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
         case RADEON_INFO_FUSION_GART_WORKING:
                 value = 1;
                 break;
+        case RADEON_INFO_BACKEND_MAP:
+                if (rdev->family >= CHIP_CAYMAN)
+                        value = rdev->config.cayman.backend_map;
+                else if (rdev->family >= CHIP_CEDAR)
+                        value = rdev->config.evergreen.backend_map;
+                else if (rdev->family >= CHIP_RV770)
+                        value = rdev->config.rv770.backend_map;
+                else if (rdev->family >= CHIP_R600)
+                        value = rdev->config.r600.backend_map;
+                else {
+                        return -EINVAL;
+                }
+                break;
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->request);
                 return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/radeon_mem.c b/drivers/gpu/drm/radeon/radeon_mem.c
index ed95155c4b1..988548efea9 100644
--- a/drivers/gpu/drm/radeon/radeon_mem.c
+++ b/drivers/gpu/drm/radeon/radeon_mem.c
@@ -139,7 +139,7 @@ static int init_heap(struct mem_block **heap, int start, int size)
         if (!blocks)
                 return -ENOMEM;
 
-        *heap = kmalloc(sizeof(**heap), GFP_KERNEL);
+        *heap = kzalloc(sizeof(**heap), GFP_KERNEL);
         if (!*heap) {
                 kfree(blocks);
                 return -ENOMEM;
@@ -150,7 +150,6 @@ static int init_heap(struct mem_block **heap, int start, int size)
         blocks->file_priv = NULL;
         blocks->next = blocks->prev = *heap;
 
-        memset(*heap, 0, sizeof(**heap));
         (*heap)->file_priv = (struct drm_file *) - 1;
         (*heap)->next = (*heap)->prev = blocks;
         return 0;
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 6df4e3cec0c..cbf80de2d9c 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -438,12 +438,16 @@ struct radeon_connector {
         struct radeon_i2c_chan *ddc_bus;
         /* some systems have an hdmi and vga port with a shared ddc line */
         bool shared_ddc;
+        /* for some Radeon chip families we apply an additional EDID header
+           check as part of the DDC probe */
+        bool requires_extended_probe;
         bool use_digital;
         /* we need to mind the EDID between detect
            and get modes due to analog/digital/tvencoder */
         struct edid *edid;
         void *con_priv;
         bool dac_load_detect;
+        bool detected_by_load; /* if the connection status was determined by load */
         uint16_t connector_object_id;
         struct radeon_hpd hpd;
         struct radeon_router router;
@@ -464,8 +468,8 @@ radeon_atombios_get_tv_info(struct radeon_device *rdev);
 extern struct drm_connector *
 radeon_get_connector_for_encoder(struct drm_encoder *encoder);
 
-extern bool radeon_encoder_is_dp_bridge(struct drm_encoder *encoder);
-extern bool radeon_connector_encoder_is_dp_bridge(struct drm_connector *connector);
+extern u16 radeon_encoder_get_dp_bridge_encoder_id(struct drm_encoder *encoder);
+extern u16 radeon_connector_encoder_get_dp_bridge_encoder_id(struct drm_connector *connector);
 extern bool radeon_connector_encoder_is_hbr2(struct drm_connector *connector);
 extern bool radeon_connector_is_dp12_capable(struct drm_connector *connector);
 
@@ -476,6 +480,7 @@ extern void radeon_dp_set_link_config(struct drm_connector *connector,
                                       struct drm_display_mode *mode);
 extern void radeon_dp_link_train(struct drm_encoder *encoder,
                                  struct drm_connector *connector);
+extern bool radeon_dp_needs_link_train(struct radeon_connector *radeon_connector);
 extern u8 radeon_dp_getsinktype(struct radeon_connector *radeon_connector);
 extern bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector);
 extern void atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode);
@@ -514,7 +519,8 @@ extern void radeon_i2c_put_byte(struct radeon_i2c_chan *i2c,
                                 u8 val);
 extern void radeon_router_select_ddc_port(struct radeon_connector *radeon_connector);
 extern void radeon_router_select_cd_port(struct radeon_connector *radeon_connector);
-extern bool radeon_ddc_probe(struct radeon_connector *radeon_connector);
+extern bool radeon_ddc_probe(struct radeon_connector *radeon_connector,
+                        bool requires_extended_probe);
 extern int radeon_ddc_get_modes(struct radeon_connector *radeon_connector);
 
 extern struct drm_encoder *radeon_best_encoder(struct drm_connector *connector);
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index aaa19dc418a..6fabe89fa6a 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -594,6 +594,9 @@ int radeon_pm_init(struct radeon_device *rdev)
                         if (rdev->pm.default_vddc)
                                 radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
                                                         SET_VOLTAGE_TYPE_ASIC_VDDC);
+                        if (rdev->pm.default_vddci)
+                                radeon_atom_set_voltage(rdev, rdev->pm.default_vddci,
+                                                        SET_VOLTAGE_TYPE_ASIC_VDDCI);
                         if (rdev->pm.default_sclk)
                                 radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
                         if (rdev->pm.default_mclk)
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index bc44a3d35ec..b4ce8645570 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -3295,7 +3295,7 @@
 #       define RADEON_RB_BUFSZ_MASK             (0x3f << 0)
 #       define RADEON_RB_BLKSZ_SHIFT            8
 #       define RADEON_RB_BLKSZ_MASK             (0x3f << 8)
-#       define RADEON_BUF_SWAP_32BIT            (1 << 17)
+#       define RADEON_BUF_SWAP_32BIT            (2 << 16)
 #       define RADEON_MAX_FETCH_SHIFT           18
 #       define RADEON_MAX_FETCH_MASK            (0x3 << 18)
 #       define RADEON_RB_NO_UPDATE              (1 << 27)
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index dee4a0c1b4b..602fa3541c4 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -40,10 +40,14 @@ void radeon_test_moves(struct radeon_device *rdev)
         size = 1024 * 1024;
 
         /* Number of tests =
-         * (Total GTT - IB pool - writeback page - ring buffer) / test size
+         * (Total GTT - IB pool - writeback page - ring buffers) / test size
          */
-        n = ((u32)(rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024 - RADEON_GPU_PAGE_SIZE -
-             rdev->cp.ring_size)) / size;
+        n = rdev->mc.gtt_size - RADEON_IB_POOL_SIZE*64*1024 - rdev->cp.ring_size;
+        if (rdev->wb.wb_obj)
+                n -= RADEON_GPU_PAGE_SIZE;
+        if (rdev->ih.ring_obj)
+                n -= rdev->ih.ring_size;
+        n /= size;
 
         gtt_obj = kzalloc(n * sizeof(*gtt_obj), GFP_KERNEL);
         if (!gtt_obj) {
@@ -132,9 +136,15 @@ void radeon_test_moves(struct radeon_device *rdev)
                      gtt_start++, vram_start++) {
                         if (*vram_start != gtt_start) {
                                 DRM_ERROR("Incorrect GTT->VRAM copy %d: Got 0x%p, "
-                                          "expected 0x%p (GTT map 0x%p-0x%p)\n",
-                                          i, *vram_start, gtt_start, gtt_map,
-                                          gtt_end);
+                                          "expected 0x%p (GTT/VRAM offset "
+                                          "0x%16llx/0x%16llx)\n",
+                                          i, *vram_start, gtt_start,
+                                          (unsigned long long)
+                                          (gtt_addr - rdev->mc.gtt_start +
+                                           (void*)gtt_start - gtt_map),
+                                          (unsigned long long)
+                                          (vram_addr - rdev->mc.vram_start +
+                                           (void*)gtt_start - gtt_map));
                                 radeon_bo_kunmap(vram_obj);
                                 goto out_cleanup;
                         }
@@ -175,9 +185,15 @@ void radeon_test_moves(struct radeon_device *rdev)
                      gtt_start++, vram_start++) {
                         if (*gtt_start != vram_start) {
                                 DRM_ERROR("Incorrect VRAM->GTT copy %d: Got 0x%p, "
-                                          "expected 0x%p (VRAM map 0x%p-0x%p)\n",
-                                          i, *gtt_start, vram_start, vram_map,
-                                          vram_end);
+                                          "expected 0x%p (VRAM/GTT offset "
+                                          "0x%16llx/0x%16llx)\n",
+                                          i, *gtt_start, vram_start,
+                                          (unsigned long long)
+                                          (vram_addr - rdev->mc.vram_start +
+                                           (void*)vram_start - vram_map),
+                                          (unsigned long long)
+                                          (gtt_addr - rdev->mc.gtt_start +
+                                           (void*)vram_start - vram_map));
                                 radeon_bo_kunmap(gtt_obj[i]);
                                 goto out_cleanup;
                         }
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 60125ddba1e..0b5468bfaf5 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -277,7 +277,12 @@ static int radeon_move_blit(struct ttm_buffer_object *bo,
                 DRM_ERROR("Trying to move memory with CP turned off.\n");
                 return -EINVAL;
         }
-        r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages, fence);
+
+        BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0);
+
+        r = radeon_copy(rdev, old_start, new_start,
+                        new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */
+                        fence);
         /* FIXME: handle copy error */
         r = ttm_bo_move_accel_cleanup(bo, (void *)fence, NULL,
                                       evict, no_wait_reserve, no_wait_gpu, new_mem);
@@ -450,6 +455,29 @@ static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
                         return -EINVAL;
                 mem->bus.base = rdev->mc.aper_base;
                 mem->bus.is_iomem = true;
+#ifdef __alpha__
+                /*
+                 * Alpha: use bus.addr to hold the ioremap() return,
+                 * so we can modify bus.base below.
+                 */
+                if (mem->placement & TTM_PL_FLAG_WC)
+                        mem->bus.addr =
+                                ioremap_wc(mem->bus.base + mem->bus.offset,
+                                           mem->bus.size);
+                else
+                        mem->bus.addr =
+                                ioremap_nocache(mem->bus.base + mem->bus.offset,
+                                                mem->bus.size);
+
+                /*
+                 * Alpha: Use just the bus offset plus
+                 * the hose/domain memory base for bus.base.
+                 * It then can be used to build PTEs for VRAM
+                 * access, as done in ttm_bo_vm_fault().
+                 */
+                mem->bus.base = (mem->bus.base & 0x0ffffffffUL) +
+                        rdev->ddev->hose->dense_mem_base;
+#endif
                 break;
         default:
                 return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/reg_srcs/cayman b/drivers/gpu/drm/radeon/reg_srcs/cayman
index 0aa8e85a945..2316977eb92 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/cayman
+++ b/drivers/gpu/drm/radeon/reg_srcs/cayman
@@ -208,6 +208,7 @@ cayman 0x9400
 0x0002834C PA_SC_VPORT_ZMAX_15
 0x00028350 SX_MISC
 0x00028354 SX_SURFACE_SYNC
+0x0002835C SX_SCATTER_EXPORT_SIZE
 0x00028380 SQ_VTX_SEMANTIC_0
 0x00028384 SQ_VTX_SEMANTIC_1
 0x00028388 SQ_VTX_SEMANTIC_2
@@ -432,6 +433,7 @@ cayman 0x9400
 0x00028700 SPI_STACK_MGMT
 0x00028704 SPI_WAVE_MGMT_1
 0x00028708 SPI_WAVE_MGMT_2
+0x00028720 GDS_ADDR_BASE
 0x00028724 GDS_ADDR_SIZE
 0x00028780 CB_BLEND0_CONTROL
 0x00028784 CB_BLEND1_CONTROL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/evergreen b/drivers/gpu/drm/radeon/reg_srcs/evergreen
index 0e28cae7ea4..161737a28c2 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/evergreen
+++ b/drivers/gpu/drm/radeon/reg_srcs/evergreen
@@ -44,6 +44,7 @@ evergreen 0x9400
 0x00008E28 SQ_STATIC_THREAD_MGMT_3
 0x00008E2C SQ_LDS_RESOURCE_MGMT
 0x00008E48 SQ_EX_ALLOC_TABLE_SLOTS
+0x00009014 SX_MEMORY_EXPORT_SIZE
 0x00009100 SPI_CONFIG_CNTL
 0x0000913C SPI_CONFIG_CNTL_1
 0x00009508 TA_CNTL_AUX
@@ -442,7 +443,9 @@ evergreen 0x9400
 0x000286EC SPI_COMPUTE_NUM_THREAD_X
 0x000286F0 SPI_COMPUTE_NUM_THREAD_Y
 0x000286F4 SPI_COMPUTE_NUM_THREAD_Z
+0x00028720 GDS_ADDR_BASE
 0x00028724 GDS_ADDR_SIZE
+0x00028728 GDS_ORDERED_WAVE_PER_SE
 0x00028780 CB_BLEND0_CONTROL
 0x00028784 CB_BLEND1_CONTROL
 0x00028788 CB_BLEND2_CONTROL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/r600 b/drivers/gpu/drm/radeon/reg_srcs/r600
index ea49752ee99..0380c5c15f8 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/r600
+++ b/drivers/gpu/drm/radeon/reg_srcs/r600
@@ -429,6 +429,7 @@ r600 0x9400
 0x00028438 SX_ALPHA_REF
 0x00028410 SX_ALPHA_TEST_CONTROL
 0x00028350 SX_MISC
+0x00009014 SX_MEMORY_EXPORT_SIZE
 0x00009604 TC_INVALIDATE
 0x00009400 TD_FILTER4
 0x00009404 TD_FILTER4_1
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index 1f5850e473c..d9b0bc4547e 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -62,6 +62,7 @@ u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 {
         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
         u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
+        int i;
 
         /* Lock the graphics update lock */
         tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
@@ -74,7 +75,11 @@ u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
                (u32)crtc_base);
 
         /* Wait for update_pending to go high. */
-        while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
+                        break;
+                udelay(1);
+        }
         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 
         /* Unlock the lock, so double-buffering can take place inside vblank */
@@ -287,6 +292,7 @@ void rs600_hpd_init(struct radeon_device *rdev)
                 default:
                         break;
                 }
+                radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
         }
         if (rdev->irq.installed)
                 rs600_irq_set(rdev);
@@ -318,10 +324,10 @@ void rs600_hpd_fini(struct radeon_device *rdev)
 
 void rs600_bm_disable(struct radeon_device *rdev)
 {
-        u32 tmp;
+        u16 tmp;
 
         /* disable bus mastering */
-        pci_read_config_word(rdev->pdev, 0x4, (u16*)&tmp);
+        pci_read_config_word(rdev->pdev, 0x4, &tmp);
         pci_write_config_word(rdev->pdev, 0x4, tmp & 0xFFFB);
         mdelay(1);
 }
@@ -530,7 +536,7 @@ int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
         addr = addr & 0xFFFFFFFFFFFFF000ULL;
         addr |= R600_PTE_VALID | R600_PTE_SYSTEM | R600_PTE_SNOOPED;
         addr |= R600_PTE_READABLE | R600_PTE_WRITEABLE;
-        writeq(addr, ((void __iomem *)ptr) + (i * 8));
+        writeq(addr, ptr + (i * 8));
         return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index 4de51891aa6..ddc206a1f61 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -47,6 +47,7 @@ u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
 {
         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
         u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
+        int i;
 
         /* Lock the graphics update lock */
         tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
@@ -66,7 +67,11 @@ u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
                (u32)crtc_base);
 
         /* Wait for update_pending to go high. */
-        while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
+                        break;
+                udelay(1);
+        }
         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 
         /* Unlock the lock, so double-buffering can take place inside vblank */
@@ -536,55 +541,6 @@ static u32 r700_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
         return backend_map;
 }
 
-static void rv770_program_channel_remap(struct radeon_device *rdev)
-{
-        u32 tcp_chan_steer, mc_shared_chremap, tmp;
-        bool force_no_swizzle;
-
-        switch (rdev->family) {
-        case CHIP_RV770:
-        case CHIP_RV730:
-                force_no_swizzle = false;
-                break;
-        case CHIP_RV710:
-        case CHIP_RV740:
-        default:
-                force_no_swizzle = true;
-                break;
-        }
-
-        tmp = RREG32(MC_SHARED_CHMAP);
-        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
-        case 0:
-        case 1:
-        default:
-                /* default mapping */
-                mc_shared_chremap = 0x00fac688;
-                break;
-        case 2:
-        case 3:
-                if (force_no_swizzle)
-                        mc_shared_chremap = 0x00fac688;
-                else
-                        mc_shared_chremap = 0x00bbc298;
-                break;
-        }
-
-        if (rdev->family == CHIP_RV740)
-                tcp_chan_steer = 0x00ef2a60;
-        else
-                tcp_chan_steer = 0x00fac688;
-
-        /* RV770 CE has special chremap setup */
-        if (rdev->pdev->device == 0x944e) {
-                tcp_chan_steer = 0x00b08b08;
-                mc_shared_chremap = 0x00b08b08;
-        }
-
-        WREG32(TCP_CHAN_STEER, tcp_chan_steer);
-        WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
-}
-
 static void rv770_gpu_init(struct radeon_device *rdev)
 {
         int i, j, num_qd_pipes;
@@ -778,14 +734,13 @@ static void rv770_gpu_init(struct radeon_device *rdev)
                                                                 (cc_rb_backend_disable >> 16));
 
         rdev->config.rv770.tile_config = gb_tiling_config;
+        rdev->config.rv770.backend_map = backend_map;
         gb_tiling_config |= BACKEND_MAP(backend_map);
 
         WREG32(GB_TILING_CONFIG, gb_tiling_config);
         WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
         WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
 
-        rv770_program_channel_remap(rdev);
-
         WREG32(CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
         WREG32(CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
         WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
diff --git a/drivers/gpu/drm/sis/sis_drv.h b/drivers/gpu/drm/sis/sis_drv.h
index ef940bad63f..194303c177a 100644
--- a/drivers/gpu/drm/sis/sis_drv.h
+++ b/drivers/gpu/drm/sis/sis_drv.h
@@ -48,8 +48,8 @@ enum sis_family {
 
 
 #define SIS_BASE (dev_priv->mmio)
-#define SIS_READ(reg)         DRM_READ32(SIS_BASE, reg);
-#define SIS_WRITE(reg, val)   DRM_WRITE32(SIS_BASE, reg, val);
+#define SIS_READ(reg)         DRM_READ32(SIS_BASE, reg)
+#define SIS_WRITE(reg, val)   DRM_WRITE32(SIS_BASE, reg, val)
 
 typedef struct drm_sis_private {
         drm_local_map_t *mmio;
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 2e618b5ac46..ef06194c5aa 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -37,7 +37,7 @@
 #include <linux/mm.h>
 #include <linux/file.h>
 #include <linux/module.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 
 #define TTM_ASSERT_LOCKED(param)
 #define TTM_DEBUG(fmt, arg...)
@@ -353,8 +353,10 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
 
                 ret = ttm_tt_set_user(bo->ttm, current,
                                       bo->buffer_start, bo->num_pages);
-                if (unlikely(ret != 0))
+                if (unlikely(ret != 0)) {
                         ttm_tt_destroy(bo->ttm);
+                        bo->ttm = NULL;
+                }
                 break;
         default:
                 printk(KERN_ERR TTM_PFX "Illegal buffer object type\n");
@@ -390,10 +392,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
          * Create and bind a ttm if required.
          */
 
-        if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm == NULL)) {
-                ret = ttm_bo_add_ttm(bo, false);
-                if (ret)
-                        goto out_err;
+        if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
+                if (bo->ttm == NULL) {
+                        bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
+                        ret = ttm_bo_add_ttm(bo, zero);
+                        if (ret)
+                                goto out_err;
+                }
 
                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
                 if (ret)
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 77dbf408c0d..082fcaea583 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -321,7 +321,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
         struct ttm_tt *ttm = bo->ttm;
         struct ttm_mem_reg *old_mem = &bo->mem;
-        struct ttm_mem_reg old_copy;
+        struct ttm_mem_reg old_copy = *old_mem;
         void *old_iomap;
         void *new_iomap;
         int ret;
@@ -635,13 +635,13 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                 if (ret)
                         return ret;
 
-                ttm_bo_free_old_node(bo);
                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
                     (bo->ttm != NULL)) {
                         ttm_tt_unbind(bo->ttm);
                         ttm_tt_destroy(bo->ttm);
                         bo->ttm = NULL;
                 }
+                ttm_bo_free_old_node(bo);
         } else {
                 /**
                  * This should help pipeline ordinary buffer moves.
diff --git a/drivers/gpu/drm/ttm/ttm_lock.c b/drivers/gpu/drm/ttm/ttm_lock.c
index de41e55a944..075daf44bce 100644
--- a/drivers/gpu/drm/ttm/ttm_lock.c
+++ b/drivers/gpu/drm/ttm/ttm_lock.c
@@ -30,7 +30,7 @@
 
 #include "ttm/ttm_lock.h"
 #include "ttm/ttm_module.h"
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 #include <linux/errno.h>
 #include <linux/wait.h>
 #include <linux/sched.h>
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index ebddd443d91..93577f2e295 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -55,7 +55,7 @@
 #include <linux/spinlock.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 
 struct ttm_object_file {
         struct ttm_object_device *tdev;
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index d948575717b..727e93daac3 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -40,7 +40,7 @@
 #include <linux/slab.h>
 #include <linux/dma-mapping.h>
 
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 
 #include "ttm/ttm_bo_driver.h"
 #include "ttm/ttm_page_alloc.h"
@@ -355,7 +355,7 @@ restart:
                         if (nr_free)
                                 goto restart;
 
-                        /* Not allowed to fall tough or break because
+                        /* Not allowed to fall through or break because
                          * following context is inside spinlock while we are
                          * outside here.
                          */
@@ -556,7 +556,7 @@ out:
 }
 
 /**
- * Fill the given pool if there isn't enough pages and requested number of
+ * Fill the given pool if there aren't enough pages and the requested number of
  * pages is small.
  */
 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
@@ -576,8 +576,8 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 
         pool->fill_lock = true;
 
-        /* If allocation request is small and there is not enough
-         * pages in pool we fill the pool first */
+        /* If allocation request is small and there are not enough
+         * pages in a pool we fill the pool up first. */
         if (count < _manager->options.small
                 && count > pool->npages) {
                 struct list_head new_pages;
@@ -614,9 +614,9 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 }
 
 /**
- * Cut count nubmer of pages from the pool and put them to return list
+ * Cut 'count' number of pages from the pool and put them on the return list.
  *
- * @return count of pages still to allocate to fill the request.
+ * @return count of pages still required to fulfill the request.
  */
 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                 struct list_head *pages, int ttm_flags,
@@ -637,7 +637,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                 goto out;
         }
         /* find the last pages to include for requested number of pages. Split
-         * pool to begin and halves to reduce search space. */
+         * pool to begin and halve it to reduce search space. */
         if (count <= pool->npages/2) {
                 i = 0;
                 list_for_each(p, &pool->list) {
@@ -651,7 +651,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                                 break;
                 }
         }
-        /* Cut count number of pages from pool */
+        /* Cut 'count' number of pages from the pool */
         list_cut_position(pages, &pool->list, p);
         pool->npages -= count;
         count = 0;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index dfe32e62bd9..8a38c91f4c9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -313,7 +313,7 @@ int vmw_framebuffer_create_handle(struct drm_framebuffer *fb,
                                   unsigned int *handle)
 {
         if (handle)
-                handle = 0;
+                *handle = 0;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
index f1a52f9e729..07ce02da78a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
@@ -585,11 +585,10 @@ int vmw_overlay_init(struct vmw_private *dev_priv)
                 return -ENOSYS;
         }
 
-        overlay = kmalloc(sizeof(*overlay), GFP_KERNEL);
+        overlay = kzalloc(sizeof(*overlay), GFP_KERNEL);
         if (!overlay)
                 return -ENOMEM;
 
-        memset(overlay, 0, sizeof(*overlay));
         mutex_init(&overlay->mutex);
         for (i = 0; i < VMW_MAX_NUM_STREAMS; i++) {
                 overlay->stream[i].buf = NULL;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index 5408b1b7996..bfe1bcce7f8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -612,11 +612,9 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
             srf->sizes[0].height == 64 &&
             srf->format == SVGA3D_A8R8G8B8) {
 
-                srf->snooper.image = kmalloc(64 * 64 * 4, GFP_KERNEL);
-                /* clear the image */
-                if (srf->snooper.image) {
-                        memset(srf->snooper.image, 0x00, 64 * 64 * 4);
-                } else {
+                /* allocate image area and clear it */
+                srf->snooper.image = kzalloc(64 * 64 * 4, GFP_KERNEL);
+                if (!srf->snooper.image) {
                         DRM_ERROR("Failed to allocate cursor_image\n");
                         ret = -ENOMEM;
                         goto out_err1;
diff --git a/drivers/gpu/ion/Kconfig b/drivers/gpu/ion/Kconfig
new file mode 100644
index 00000000000..9a8cbdd9836
--- /dev/null
+++ b/drivers/gpu/ion/Kconfig
@@ -0,0 +1,17 @@
+menuconfig ION
+        tristate "Ion Memory Manager"
+        select GENERIC_ALLOCATOR
+        help
+          Chose this option to enable the ION Memory Manager.
+
+config ION_IOMMU
+       bool
+
+config ION_TEGRA
+        tristate "Ion for Tegra"
+        depends on ARCH_TEGRA && ION
+        select TEGRA_IOMMU_SMMU if !ARCH_TEGRA_2x_SOC
+        select ION_IOMMU if TEGRA_IOMMU_SMMU
+        help
+          Choose this option if you wish to use ion on an nVidia Tegra.
+
diff --git a/drivers/gpu/ion/Makefile b/drivers/gpu/ion/Makefile
new file mode 100644
index 00000000000..4ddc78e9d41
--- /dev/null
+++ b/drivers/gpu/ion/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_ION) +=    ion.o ion_heap.o ion_system_heap.o ion_carveout_heap.o
+obj-$(CONFIG_ION_IOMMU) += ion_iommu_heap.o
+obj-$(CONFIG_ION_TEGRA) += tegra/
diff --git a/drivers/gpu/ion/ion.c b/drivers/gpu/ion/ion.c
new file mode 100644
index 00000000000..512ebc5cc8e
--- /dev/null
+++ b/drivers/gpu/ion/ion.c
@@ -0,0 +1,1152 @@
+/*
+ * drivers/gpu/ion/ion.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt)     "%s():%d: " fmt, __func__, __LINE__
+
+#include <linux/device.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/anon_inodes.h>
+#include <linux/ion.h>
+#include <linux/list.h>
+#include <linux/miscdevice.h>
+#include <linux/mm.h>
+#include <linux/mm_types.h>
+#include <linux/rbtree.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#include <linux/debugfs.h>
+
+#include "ion_priv.h"
+#define DEBUG
+
+/* this function should only be called while dev->lock is held */
+static void ion_buffer_add(struct ion_device *dev,
+                           struct ion_buffer *buffer)
+{
+        struct rb_node **p = &dev->buffers.rb_node;
+        struct rb_node *parent = NULL;
+        struct ion_buffer *entry;
+
+        while (*p) {
+                parent = *p;
+                entry = rb_entry(parent, struct ion_buffer, node);
+
+                if (buffer < entry) {
+                        p = &(*p)->rb_left;
+                } else if (buffer > entry) {
+                        p = &(*p)->rb_right;
+                } else {
+                        pr_err("buffer already found.");
+                        BUG();
+                }
+        }
+
+        rb_link_node(&buffer->node, parent, p);
+        rb_insert_color(&buffer->node, &dev->buffers);
+}
+
+/* this function should only be called while dev->lock is held */
+static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
+                                     struct ion_device *dev,
+                                     unsigned long len,
+                                     unsigned long align,
+                                     unsigned long flags)
+{
+        struct ion_buffer *buffer;
+        int ret;
+
+        buffer = kzalloc(sizeof(struct ion_buffer), GFP_KERNEL);
+        if (!buffer)
+                return ERR_PTR(-ENOMEM);
+
+        buffer->heap = heap;
+        kref_init(&buffer->ref);
+
+        ret = heap->ops->allocate(heap, buffer, len, align, flags);
+        if (ret) {
+                kfree(buffer);
+                return ERR_PTR(ret);
+        }
+        buffer->dev = dev;
+        buffer->size = len;
+        mutex_init(&buffer->lock);
+        ion_buffer_add(dev, buffer);
+        return buffer;
+}
+
+static void ion_buffer_destroy(struct kref *kref)
+{
+        struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);
+        struct ion_device *dev = buffer->dev;
+
+        buffer->heap->ops->free(buffer);
+        mutex_lock(&dev->lock);
+        rb_erase(&buffer->node, &dev->buffers);
+        mutex_unlock(&dev->lock);
+        kfree(buffer);
+}
+
+void ion_buffer_get(struct ion_buffer *buffer)
+{
+        kref_get(&buffer->ref);
+}
+
+static int ion_buffer_put(struct ion_buffer *buffer)
+{
+        return kref_put(&buffer->ref, ion_buffer_destroy);
+}
+
+struct ion_handle *ion_handle_create(struct ion_client *client,
+                                     struct ion_buffer *buffer)
+{
+        struct ion_handle *handle;
+
+        handle = kzalloc(sizeof(struct ion_handle), GFP_KERNEL);
+        if (!handle)
+                return ERR_PTR(-ENOMEM);
+        kref_init(&handle->ref);
+        rb_init_node(&handle->node);
+        handle->client = client;
+        ion_buffer_get(buffer);
+        handle->buffer = buffer;
+
+        return handle;
+}
+
+static void ion_handle_destroy(struct kref *kref)
+{
+        struct ion_handle *handle = container_of(kref, struct ion_handle, ref);
+        /* XXX Can a handle be destroyed while it's map count is non-zero?:
+           if (handle->map_cnt) unmap
+         */
+        ion_buffer_put(handle->buffer);
+        mutex_lock(&handle->client->lock);
+        if (!RB_EMPTY_NODE(&handle->node))
+                rb_erase(&handle->node, &handle->client->handles);
+        mutex_unlock(&handle->client->lock);
+        kfree(handle);
+}
+
+struct ion_buffer *ion_handle_buffer(struct ion_handle *handle)
+{
+        return handle->buffer;
+}
+
+void ion_handle_get(struct ion_handle *handle)
+{
+        kref_get(&handle->ref);
+}
+
+int ion_handle_put(struct ion_handle *handle)
+{
+        return kref_put(&handle->ref, ion_handle_destroy);
+}
+
+static struct ion_handle *ion_handle_lookup(struct ion_client *client,
+                                            struct ion_buffer *buffer)
+{
+        struct rb_node *n;
+
+        for (n = rb_first(&client->handles); n; n = rb_next(n)) {
+                struct ion_handle *handle = rb_entry(n, struct ion_handle,
+                                                     node);
+                if (handle->buffer == buffer)
+                        return handle;
+        }
+        return NULL;
+}
+
+bool ion_handle_validate(struct ion_client *client, struct ion_handle *handle)
+{
+        struct rb_node *n = client->handles.rb_node;
+
+        while (n) {
+                struct ion_handle *handle_node = rb_entry(n, struct ion_handle,
+                                                          node);
+                if (handle < handle_node)
+                        n = n->rb_left;
+                else if (handle > handle_node)
+                        n = n->rb_right;
+                else
+                        return true;
+        }
+        WARN(1, "invalid handle passed h=0x%x,comm=%d\n", handle,
+                current->group_leader->comm);
+        return false;
+}
+
+void ion_handle_add(struct ion_client *client, struct ion_handle *handle)
+{
+        struct rb_node **p = &client->handles.rb_node;
+        struct rb_node *parent = NULL;
+        struct ion_handle *entry;
+
+        while (*p) {
+                parent = *p;
+                entry = rb_entry(parent, struct ion_handle, node);
+
+                if (handle < entry)
+                        p = &(*p)->rb_left;
+                else if (handle > entry)
+                        p = &(*p)->rb_right;
+                else
+                        WARN(1, "%s: buffer already found.", __func__);
+        }
+
+        rb_link_node(&handle->node, parent, p);
+        rb_insert_color(&handle->node, &client->handles);
+}
+
+struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
+                             size_t align, unsigned int flags)
+{
+        struct rb_node *n;
+        struct ion_handle *handle;
+        struct ion_device *dev = client->dev;
+        struct ion_buffer *buffer = NULL;
+
+        /*
+         * traverse the list of heaps available in this system in priority
+         * order.  If the heap type is supported by the client, and matches the
+         * request of the caller allocate from it.  Repeat until allocate has
+         * succeeded or all heaps have been tried
+         */
+        mutex_lock(&dev->lock);
+        for (n = rb_first(&dev->heaps); n != NULL; n = rb_next(n)) {
+                struct ion_heap *heap = rb_entry(n, struct ion_heap, node);
+                /* if the client doesn't support this heap type */
+                if (!((1 << heap->type) & client->heap_mask))
+                        continue;
+                /* if the caller didn't specify this heap type */
+                if (!((1 << heap->id) & flags))
+                        continue;
+                buffer = ion_buffer_create(heap, dev, len, align, flags);
+                if (!IS_ERR_OR_NULL(buffer))
+                        break;
+        }
+        mutex_unlock(&dev->lock);
+
+        if (IS_ERR_OR_NULL(buffer))
+                return ERR_PTR(PTR_ERR(buffer));
+
+        handle = ion_handle_create(client, buffer);
+
+        if (IS_ERR_OR_NULL(handle))
+                goto end;
+
+        /*
+         * ion_buffer_create will create a buffer with a ref_cnt of 1,
+         * and ion_handle_create will take a second reference, drop one here
+         */
+        ion_buffer_put(buffer);
+
+        mutex_lock(&client->lock);
+        ion_handle_add(client, handle);
+        mutex_unlock(&client->lock);
+        return handle;
+
+end:
+        ion_buffer_put(buffer);
+        return handle;
+}
+
+void ion_free(struct ion_client *client, struct ion_handle *handle)
+{
+        bool valid_handle;
+
+        BUG_ON(client != handle->client);
+
+        mutex_lock(&client->lock);
+        valid_handle = ion_handle_validate(client, handle);
+        mutex_unlock(&client->lock);
+
+        if (!valid_handle) {
+                WARN(1, "%s: invalid handle passed to free.\n", __func__);
+                return;
+        }
+        ion_handle_put(handle);
+}
+
+static bool _ion_map(int *buffer_cnt, int *handle_cnt)
+{
+        bool map;
+
+        BUG_ON(*handle_cnt != 0 && *buffer_cnt == 0);
+
+        if (*buffer_cnt)
+                map = false;
+        else
+                map = true;
+        if (*handle_cnt == 0)
+                (*buffer_cnt)++;
+        (*handle_cnt)++;
+        return map;
+}
+
+static bool _ion_unmap(int *buffer_cnt, int *handle_cnt)
+{
+        BUG_ON(*handle_cnt == 0);
+        (*handle_cnt)--;
+        if (*handle_cnt != 0)
+                return false;
+        BUG_ON(*buffer_cnt == 0);
+        (*buffer_cnt)--;
+        if (*buffer_cnt == 0)
+                return true;
+        return false;
+}
+
+int ion_phys(struct ion_client *client, struct ion_handle *handle,
+             ion_phys_addr_t *addr, size_t *len)
+{
+        struct ion_buffer *buffer;
+        int ret;
+
+        mutex_lock(&client->lock);
+        if (!ion_handle_validate(client, handle)) {
+                mutex_unlock(&client->lock);
+                return -EINVAL;
+        }
+
+        buffer = handle->buffer;
+
+        if (!buffer->heap->ops->phys) {
+                pr_err("ion_phys is not implemented by this heap.\n");
+                mutex_unlock(&client->lock);
+                return -ENODEV;
+        }
+        mutex_unlock(&client->lock);
+        ret = buffer->heap->ops->phys(buffer->heap, buffer, addr, len);
+        return ret;
+}
+
+void *ion_map_kernel(struct ion_client *client, struct ion_handle *handle)
+{
+        struct ion_buffer *buffer;
+        void *vaddr;
+
+        mutex_lock(&client->lock);
+        if (!ion_handle_validate(client, handle)) {
+                WARN(1, "invalid handle passed to map_kernel.\n");
+                mutex_unlock(&client->lock);
+                return ERR_PTR(-EINVAL);
+        }
+
+        buffer = handle->buffer;
+        mutex_lock(&buffer->lock);
+
+        if (!handle->buffer->heap->ops->map_kernel) {
+                pr_err("map_kernel is not implemented by this heap.\n");
+                mutex_unlock(&buffer->lock);
+                mutex_unlock(&client->lock);
+                return ERR_PTR(-ENODEV);
+        }
+
+        if (_ion_map(&buffer->kmap_cnt, &handle->kmap_cnt)) {
+                vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
+                if (IS_ERR_OR_NULL(vaddr))
+                        _ion_unmap(&buffer->kmap_cnt, &handle->kmap_cnt);
+                buffer->vaddr = vaddr;
+        } else {
+                vaddr = buffer->vaddr;
+        }
+        mutex_unlock(&buffer->lock);
+        mutex_unlock(&client->lock);
+        return vaddr;
+}
+
+struct scatterlist *ion_map_dma(struct ion_client *client,
+                                struct ion_handle *handle)
+{
+        struct ion_buffer *buffer;
+        struct scatterlist *sglist;
+
+        mutex_lock(&client->lock);
+        if (!ion_handle_validate(client, handle)) {
+                WARN(1, "invalid handle passed to map_dma.\n");
+                mutex_unlock(&client->lock);
+                return ERR_PTR(-EINVAL);
+        }
+        buffer = handle->buffer;
+        mutex_lock(&buffer->lock);
+
+        if (!handle->buffer->heap->ops->map_dma) {
+                pr_err("map_kernel is not implemented by this heap.\n");
+                mutex_unlock(&buffer->lock);
+                mutex_unlock(&client->lock);
+                return ERR_PTR(-ENODEV);
+        }
+        if (_ion_map(&buffer->dmap_cnt, &handle->dmap_cnt)) {
+                sglist = buffer->heap->ops->map_dma(buffer->heap, buffer);
+                if (IS_ERR_OR_NULL(sglist))
+                        _ion_unmap(&buffer->dmap_cnt, &handle->dmap_cnt);
+                buffer->sglist = sglist;
+        } else {
+                sglist = buffer->sglist;
+        }
+        mutex_unlock(&buffer->lock);
+        mutex_unlock(&client->lock);
+        return sglist;
+}
+
+struct scatterlist *iommu_heap_remap_dma(struct ion_heap *heap,
+                                              struct ion_buffer *buf,
+                                              unsigned long addr);
+int ion_remap_dma(struct ion_client *client,
+                        struct ion_handle *handle,
+                        unsigned long addr)
+{
+        struct ion_buffer *buffer;
+        int ret;
+
+        mutex_lock(&client->lock);
+        if (!ion_handle_validate(client, handle)) {
+                pr_err("invalid handle passed to map_dma.\n");
+                mutex_unlock(&client->lock);
+                return -EINVAL;
+        }
+        buffer = handle->buffer;
+        mutex_lock(&buffer->lock);
+
+        ret = iommu_heap_remap_dma(buffer->heap, buffer, addr);
+
+        mutex_unlock(&buffer->lock);
+        mutex_unlock(&client->lock);
+        return ret;
+}
+
+void ion_unmap_kernel(struct ion_client *client, struct ion_handle *handle)
+{
+        struct ion_buffer *buffer;
+
+        mutex_lock(&client->lock);
+        buffer = handle->buffer;
+        mutex_lock(&buffer->lock);
+        if (_ion_unmap(&buffer->kmap_cnt, &handle->kmap_cnt)) {
+                buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
+                buffer->vaddr = NULL;
+        }
+        mutex_unlock(&buffer->lock);
+        mutex_unlock(&client->lock);
+}
+
+void ion_unmap_dma(struct ion_client *client, struct ion_handle *handle)
+{
+        struct ion_buffer *buffer;
+
+        mutex_lock(&client->lock);
+        buffer = handle->buffer;
+        mutex_lock(&buffer->lock);
+        if (_ion_unmap(&buffer->dmap_cnt, &handle->dmap_cnt)) {
+                buffer->heap->ops->unmap_dma(buffer->heap, buffer);
+                buffer->sglist = NULL;
+        }
+        mutex_unlock(&buffer->lock);
+        mutex_unlock(&client->lock);
+}
+
+
+struct ion_buffer *ion_share(struct ion_client *client,
+                                 struct ion_handle *handle)
+{
+        bool valid_handle;
+
+        mutex_lock(&client->lock);
+        valid_handle = ion_handle_validate(client, handle);
+        mutex_unlock(&client->lock);
+        if (!valid_handle) {
+                WARN(1, "%s: invalid handle passed to share.\n", __func__);
+                return ERR_PTR(-EINVAL);
+        }
+
+        /* do not take an extra reference here, the burden is on the caller
+         * to make sure the buffer doesn't go away while it's passing it
+         * to another client -- ion_free should not be called on this handle
+         * until the buffer has been imported into the other client
+         */
+        return handle->buffer;
+}
+
+struct ion_handle *ion_import(struct ion_client *client,
+                              struct ion_buffer *buffer)
+{
+        struct ion_handle *handle = NULL;
+
+        mutex_lock(&client->lock);
+        /* if a handle exists for this buffer just take a reference to it */
+        handle = ion_handle_lookup(client, buffer);
+        if (!IS_ERR_OR_NULL(handle)) {
+                ion_handle_get(handle);
+                goto end;
+        }
+        handle = ion_handle_create(client, buffer);
+        if (IS_ERR_OR_NULL(handle)) {
+                pr_err("error during handle create\n");
+                goto end;
+        }
+        ion_handle_add(client, handle);
+end:
+        mutex_unlock(&client->lock);
+        return handle;
+}
+
+static const struct file_operations ion_share_fops;
+
+struct ion_handle *ion_import_fd(struct ion_client *client, int fd)
+{
+        struct file *file = fget(fd);
+        struct ion_handle *handle;
+
+        if (!file) {
+                pr_err("imported fd not found in file table.\n");
+                return ERR_PTR(-EINVAL);
+        }
+        if (file->f_op != &ion_share_fops) {
+                pr_err("imported file is not a shared ion file.\n");
+                handle = ERR_PTR(-EINVAL);
+                goto end;
+        }
+        handle = ion_import(client, file->private_data);
+end:
+        fput(file);
+        return handle;
+}
+
+static int ion_debug_client_show(struct seq_file *s, void *unused)
+{
+        struct ion_client *client = s->private;
+        struct rb_node *n;
+        size_t sizes[ION_NUM_HEAPS] = {0};
+        const char *names[ION_NUM_HEAPS] = {0};
+        int i;
+
+        mutex_lock(&client->lock);
+        for (n = rb_first(&client->handles); n; n = rb_next(n)) {
+                struct ion_handle *handle = rb_entry(n, struct ion_handle,
+                                                     node);
+                enum ion_heap_type type = handle->buffer->heap->type;
+
+                if (!names[type])
+                        names[type] = handle->buffer->heap->name;
+                sizes[type] += handle->buffer->size;
+        }
+        mutex_unlock(&client->lock);
+
+        seq_printf(s, "%16.16s: %16.16s\n", "heap_name", "size_in_bytes");
+        for (i = 0; i < ION_NUM_HEAPS; i++) {
+                if (!names[i])
+                        continue;
+                seq_printf(s, "%16.16s: %16u %d\n", names[i], sizes[i],
+                           atomic_read(&client->ref.refcount));
+        }
+        return 0;
+}
+
+static int ion_debug_client_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, ion_debug_client_show, inode->i_private);
+}
+
+static const struct file_operations debug_client_fops = {
+        .open = ion_debug_client_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static struct ion_client *ion_client_lookup(struct ion_device *dev,
+                                            struct task_struct *task)
+{
+        struct rb_node *n = dev->user_clients.rb_node;
+        struct ion_client *client;
+
+        mutex_lock(&dev->lock);
+        while (n) {
+                client = rb_entry(n, struct ion_client, node);
+                if (task == client->task) {
+                        ion_client_get(client);
+                        mutex_unlock(&dev->lock);
+                        return client;
+                } else if (task < client->task) {
+                        n = n->rb_left;
+                } else if (task > client->task) {
+                        n = n->rb_right;
+                }
+        }
+        mutex_unlock(&dev->lock);
+        return NULL;
+}
+
+struct ion_client *ion_client_create(struct ion_device *dev,
+                                     unsigned int heap_mask,
+                                     const char *name)
+{
+        struct ion_client *client;
+        struct task_struct *task;
+        struct rb_node **p;
+        struct rb_node *parent = NULL;
+        struct ion_client *entry;
+        char debug_name[64];
+        pid_t pid;
+
+        get_task_struct(current->group_leader);
+        task_lock(current->group_leader);
+        pid = task_pid_nr(current->group_leader);
+        /* don't bother to store task struct for kernel threads,
+           they can't be killed anyway */
+        if (current->group_leader->flags & PF_KTHREAD) {
+                put_task_struct(current->group_leader);
+                task = NULL;
+        } else {
+                task = current->group_leader;
+        }
+        task_unlock(current->group_leader);
+
+        /* if this isn't a kernel thread, see if a client already
+           exists */
+        if (task) {
+                client = ion_client_lookup(dev, task);
+                if (!IS_ERR_OR_NULL(client)) {
+                        put_task_struct(current->group_leader);
+                        return client;
+                }
+        }
+
+        client = kzalloc(sizeof(struct ion_client), GFP_KERNEL);
+        if (!client) {
+                put_task_struct(current->group_leader);
+                return ERR_PTR(-ENOMEM);
+        }
+
+        client->dev = dev;
+        client->handles = RB_ROOT;
+        mutex_init(&client->lock);
+        client->name = name;
+        client->heap_mask = heap_mask;
+        client->task = task;
+        client->pid = pid;
+        kref_init(&client->ref);
+
+        mutex_lock(&dev->lock);
+        if (task) {
+                p = &dev->user_clients.rb_node;
+                while (*p) {
+                        parent = *p;
+                        entry = rb_entry(parent, struct ion_client, node);
+
+                        if (task < entry->task)
+                                p = &(*p)->rb_left;
+                        else if (task > entry->task)
+                                p = &(*p)->rb_right;
+                }
+                rb_link_node(&client->node, parent, p);
+                rb_insert_color(&client->node, &dev->user_clients);
+        } else {
+                p = &dev->kernel_clients.rb_node;
+                while (*p) {
+                        parent = *p;
+                        entry = rb_entry(parent, struct ion_client, node);
+
+                        if (client < entry)
+                                p = &(*p)->rb_left;
+                        else if (client > entry)
+                                p = &(*p)->rb_right;
+                }
+                rb_link_node(&client->node, parent, p);
+                rb_insert_color(&client->node, &dev->kernel_clients);
+        }
+
+        snprintf(debug_name, 64, "%u", client->pid);
+        client->debug_root = debugfs_create_file(debug_name, 0664,
+                                                 dev->debug_root, client,
+                                                 &debug_client_fops);
+        mutex_unlock(&dev->lock);
+
+        return client;
+}
+
+static void _ion_client_destroy(struct kref *kref)
+{
+        struct ion_client *client = container_of(kref, struct ion_client, ref);
+        struct ion_device *dev = client->dev;
+        struct rb_node *n;
+
+        pr_debug("\n");
+        while ((n = rb_first(&client->handles))) {
+                struct ion_handle *handle = rb_entry(n, struct ion_handle,
+                                                     node);
+                ion_handle_destroy(&handle->ref);
+        }
+        mutex_lock(&dev->lock);
+        if (client->task) {
+                rb_erase(&client->node, &dev->user_clients);
+                put_task_struct(client->task);
+        } else {
+                rb_erase(&client->node, &dev->kernel_clients);
+        }
+        debugfs_remove_recursive(client->debug_root);
+        mutex_unlock(&dev->lock);
+
+        kfree(client);
+}
+
+void ion_client_get(struct ion_client *client)
+{
+        kref_get(&client->ref);
+}
+
+int ion_client_put(struct ion_client *client)
+{
+        return kref_put(&client->ref, _ion_client_destroy);
+}
+
+void ion_client_destroy(struct ion_client *client)
+{
+        ion_client_put(client);
+}
+
+static int ion_share_release(struct inode *inode, struct file* file)
+{
+        struct ion_buffer *buffer = file->private_data;
+
+        pr_debug("\n");
+        /* drop the reference to the buffer -- this prevents the
+           buffer from going away because the client holding it exited
+           while it was being passed */
+        ion_buffer_put(buffer);
+        return 0;
+}
+
+static void ion_vma_open(struct vm_area_struct *vma)
+{
+
+        struct ion_buffer *buffer = vma->vm_file->private_data;
+        struct ion_handle *handle = vma->vm_private_data;
+        struct ion_client *client;
+
+        pr_debug("\n");
+        /* check that the client still exists and take a reference so
+           it can't go away until this vma is closed */
+        client = ion_client_lookup(buffer->dev, current->group_leader);
+        if (IS_ERR_OR_NULL(client)) {
+                vma->vm_private_data = NULL;
+                return;
+        }
+        ion_buffer_get(buffer);
+        ion_handle_get(handle);
+        pr_debug("client_cnt %d handle_cnt %d alloc_cnt %d\n",
+                 atomic_read(&client->ref.refcount),
+                 atomic_read(&handle->ref.refcount),
+                 atomic_read(&buffer->ref.refcount));
+}
+
+static void ion_vma_close(struct vm_area_struct *vma)
+{
+        struct ion_handle *handle = vma->vm_private_data;
+        struct ion_buffer *buffer = vma->vm_file->private_data;
+        struct ion_client *client;
+
+        pr_debug("\n");
+        /* this indicates the client is gone, nothing to do here */
+        if (!handle)
+                return;
+        client = handle->client;
+        pr_debug("client_cnt %d handle_cnt %d alloc_cnt %d\n",
+                 atomic_read(&client->ref.refcount),
+                 atomic_read(&handle->ref.refcount),
+                 atomic_read(&buffer->ref.refcount));
+        ion_handle_put(handle);
+        ion_client_put(client);
+        ion_buffer_put(buffer);
+        pr_debug("client_cnt %d handle_cnt %d alloc_cnt %d\n",
+                 atomic_read(&client->ref.refcount),
+                 atomic_read(&handle->ref.refcount),
+                 atomic_read(&buffer->ref.refcount));
+}
+
+static struct vm_operations_struct ion_vm_ops = {
+        .open = ion_vma_open,
+        .close = ion_vma_close,
+};
+
+static int ion_share_mmap(struct file *file, struct vm_area_struct *vma)
+{
+        struct ion_buffer *buffer = file->private_data;
+        unsigned long size = vma->vm_end - vma->vm_start;
+        struct ion_client *client;
+        struct ion_handle *handle;
+        int ret;
+
+        pr_debug("\n");
+        /* make sure the client still exists, it's possible for the client to
+           have gone away but the map/share fd still to be around, take
+           a reference to it so it can't go away while this mapping exists */
+        client = ion_client_lookup(buffer->dev, current->group_leader);
+        if (IS_ERR_OR_NULL(client)) {
+                WARN(1, "trying to mmap an ion handle in a process with no "
+                       "ion client\n");
+                return -EINVAL;
+        }
+
+        if ((size > buffer->size) || (size + (vma->vm_pgoff << PAGE_SHIFT) >
+                                     buffer->size)) {
+                WARN(1, "trying to map larger area than handle has available"
+                       "\n");
+                ret = -EINVAL;
+                goto err;
+        }
+
+        /* find the handle and take a reference to it */
+        handle = ion_import(client, buffer);
+        if (IS_ERR_OR_NULL(handle)) {
+                ret = -EINVAL;
+                goto err;
+        }
+        ion_buffer_get(buffer);
+
+        if (!handle->buffer->heap->ops->map_user) {
+                pr_err("this heap does not define a method for mapping "
+                       "to userspace\n");
+                ret = -EINVAL;
+                goto err1;
+        }
+
+        mutex_lock(&buffer->lock);
+        /* now map it to userspace */
+        ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
+        mutex_unlock(&buffer->lock);
+        if (ret) {
+                pr_err("failure mapping buffer to userspace\n");
+                goto err1;
+        }
+
+        vma->vm_ops = &ion_vm_ops;
+        /* move the handle into the vm_private_data so we can access it from
+           vma_open/close */
+        vma->vm_private_data = handle;
+        pr_debug("client_cnt %d handle_cnt %d alloc_cnt %d\n",
+                 atomic_read(&client->ref.refcount),
+                 atomic_read(&handle->ref.refcount),
+                 atomic_read(&buffer->ref.refcount));
+        return 0;
+
+err1:
+        /* drop the reference to the handle */
+        ion_handle_put(handle);
+err:
+        /* drop the reference to the client */
+        ion_client_put(client);
+        return ret;
+}
+
+static const struct file_operations ion_share_fops = {
+        .owner          = THIS_MODULE,
+        .release        = ion_share_release,
+        .mmap           = ion_share_mmap,
+};
+
+static int ion_ioctl_share(struct file *parent, struct ion_client *client,
+                           struct ion_handle *handle)
+{
+        int fd = get_unused_fd();
+        struct file *file;
+
+        if (fd < 0)
+                return -ENFILE;
+
+        file = anon_inode_getfile("ion_share_fd", &ion_share_fops,
+                                  handle->buffer, O_RDWR);
+        if (IS_ERR_OR_NULL(file))
+                goto err;
+        ion_buffer_get(handle->buffer);
+        fd_install(fd, file);
+
+        return fd;
+
+err:
+        put_unused_fd(fd);
+        return -ENFILE;
+}
+
+static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+        struct ion_client *client = filp->private_data;
+
+        switch (cmd) {
+        case ION_IOC_ALLOC:
+        {
+                struct ion_allocation_data data;
+
+                if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                        return -EFAULT;
+                data.handle = ion_alloc(client, data.len, data.align,
+                                             data.flags);
+                if (copy_to_user((void __user *)arg, &data, sizeof(data)))
+                        return -EFAULT;
+                break;
+        }
+        case ION_IOC_FREE:
+        {
+                struct ion_handle_data data;
+                bool valid;
+
+                if (copy_from_user(&data, (void __user *)arg,
+                                   sizeof(struct ion_handle_data)))
+                        return -EFAULT;
+                mutex_lock(&client->lock);
+                valid = ion_handle_validate(client, data.handle);
+                mutex_unlock(&client->lock);
+                if (!valid)
+                        return -EINVAL;
+                ion_free(client, data.handle);
+                break;
+        }
+        case ION_IOC_MAP:
+        case ION_IOC_SHARE:
+        {
+                struct ion_fd_data data;
+
+                if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                        return -EFAULT;
+                mutex_lock(&client->lock);
+                if (!ion_handle_validate(client, data.handle)) {
+                        WARN(1, "invalid handle passed to share ioctl.\n");
+                        mutex_unlock(&client->lock);
+                        return -EINVAL;
+                }
+                data.fd = ion_ioctl_share(filp, client, data.handle);
+                mutex_unlock(&client->lock);
+                if (copy_to_user((void __user *)arg, &data, sizeof(data)))
+                        return -EFAULT;
+                break;
+        }
+        case ION_IOC_IMPORT:
+        {
+                struct ion_fd_data data;
+                if (copy_from_user(&data, (void __user *)arg,
+                                   sizeof(struct ion_fd_data)))
+                        return -EFAULT;
+
+                data.handle = ion_import_fd(client, data.fd);
+                if (IS_ERR(data.handle))
+                        data.handle = NULL;
+                if (copy_to_user((void __user *)arg, &data,
+                                 sizeof(struct ion_fd_data)))
+                        return -EFAULT;
+                break;
+        }
+        case ION_IOC_CUSTOM:
+        {
+                struct ion_device *dev = client->dev;
+                struct ion_custom_data data;
+
+                if (!dev->custom_ioctl)
+                        return -ENOTTY;
+                if (copy_from_user(&data, (void __user *)arg,
+                                sizeof(struct ion_custom_data)))
+                        return -EFAULT;
+                return dev->custom_ioctl(client, data.cmd, data.arg);
+        }
+        default:
+                return -ENOTTY;
+        }
+        return 0;
+}
+
+static int ion_release(struct inode *inode, struct file *file)
+{
+        struct ion_client *client = file->private_data;
+
+        pr_debug("\n");
+        ion_client_put(client);
+        return 0;
+}
+
+static int ion_open(struct inode *inode, struct file *file)
+{
+        struct miscdevice *miscdev = file->private_data;
+        struct ion_device *dev = container_of(miscdev, struct ion_device, dev);
+        struct ion_client *client;
+
+        pr_debug("\n");
+        client = ion_client_create(dev, -1, "user");
+        if (IS_ERR_OR_NULL(client))
+                return PTR_ERR(client);
+        file->private_data = client;
+
+        return 0;
+}
+
+static const struct file_operations ion_fops = {
+        .owner          = THIS_MODULE,
+        .open           = ion_open,
+        .release        = ion_release,
+        .unlocked_ioctl = ion_ioctl,
+};
+
+static size_t ion_debug_heap_total(struct ion_client *client,
+                                   enum ion_heap_type type)
+{
+        size_t size = 0;
+        struct rb_node *n;
+
+        mutex_lock(&client->lock);
+        for (n = rb_first(&client->handles); n; n = rb_next(n)) {
+                struct ion_handle *handle = rb_entry(n,
+                                                     struct ion_handle,
+                                                     node);
+                if (handle->buffer->heap->type == type)
+                        size += handle->buffer->size;
+        }
+        mutex_unlock(&client->lock);
+        return size;
+}
+
+static int ion_debug_heap_show(struct seq_file *s, void *unused)
+{
+        struct ion_heap *heap = s->private;
+        struct ion_device *dev = heap->dev;
+        struct rb_node *n;
+
+        seq_printf(s, "%16.s %16.s %16.s\n", "client", "pid", "size");
+        for (n = rb_first(&dev->user_clients); n; n = rb_next(n)) {
+                struct ion_client *client = rb_entry(n, struct ion_client,
+                                                     node);
+                char task_comm[TASK_COMM_LEN];
+                size_t size = ion_debug_heap_total(client, heap->type);
+                if (!size)
+                        continue;
+
+                get_task_comm(task_comm, client->task);
+                seq_printf(s, "%16.s %16u %16u\n", task_comm, client->pid,
+                           size);
+        }
+
+        for (n = rb_first(&dev->kernel_clients); n; n = rb_next(n)) {
+                struct ion_client *client = rb_entry(n, struct ion_client,
+                                                     node);
+                size_t size = ion_debug_heap_total(client, heap->type);
+                if (!size)
+                        continue;
+                seq_printf(s, "%16.s %16u %16u\n", client->name, client->pid,
+                           size);
+        }
+        return 0;
+}
+
+static int ion_debug_heap_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, ion_debug_heap_show, inode->i_private);
+}
+
+static const struct file_operations debug_heap_fops = {
+        .open = ion_debug_heap_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
+{
+        struct rb_node **p = &dev->heaps.rb_node;
+        struct rb_node *parent = NULL;
+        struct ion_heap *entry;
+
+        heap->dev = dev;
+        mutex_lock(&dev->lock);
+        while (*p) {
+                parent = *p;
+                entry = rb_entry(parent, struct ion_heap, node);
+
+                if (heap->id < entry->id) {
+                        p = &(*p)->rb_left;
+                } else if (heap->id > entry->id ) {
+                        p = &(*p)->rb_right;
+                } else {
+                        pr_err("can not insert multiple heaps with "
+                                "id %d\n", heap->id);
+                        goto end;
+                }
+        }
+
+        rb_link_node(&heap->node, parent, p);
+        rb_insert_color(&heap->node, &dev->heaps);
+        debugfs_create_file(heap->name, 0664, dev->debug_root, heap,
+                            &debug_heap_fops);
+end:
+        mutex_unlock(&dev->lock);
+}
+
+struct ion_device *ion_device_create(long (*custom_ioctl)
+                                     (struct ion_client *client,
+                                      unsigned int cmd,
+                                      unsigned long arg))
+{
+        struct ion_device *idev;
+        int ret;
+
+        idev = kzalloc(sizeof(struct ion_device), GFP_KERNEL);
+        if (!idev)
+                return ERR_PTR(-ENOMEM);
+
+        idev->dev.minor = MISC_DYNAMIC_MINOR;
+        idev->dev.name = "ion";
+        idev->dev.fops = &ion_fops;
+        idev->dev.parent = NULL;
+        ret = misc_register(&idev->dev);
+        if (ret) {
+                pr_err("ion: failed to register misc device.\n");
+                return ERR_PTR(ret);
+        }
+
+        idev->debug_root = debugfs_create_dir("ion", NULL);
+        if (IS_ERR_OR_NULL(idev->debug_root))
+                pr_err("ion: failed to create debug files.\n");
+
+        idev->custom_ioctl = custom_ioctl;
+        idev->buffers = RB_ROOT;
+        mutex_init(&idev->lock);
+        idev->heaps = RB_ROOT;
+        idev->user_clients = RB_ROOT;
+        idev->kernel_clients = RB_ROOT;
+        return idev;
+}
+
+void ion_device_destroy(struct ion_device *dev)
+{
+        misc_deregister(&dev->dev);
+        /* XXX need to free the heaps and clients ? */
+        kfree(dev);
+}
+
+struct ion_client *ion_client_get_file(int fd)
+{
+        struct ion_client *client = ERR_PTR(-EFAULT);
+        struct file *f = fget(fd);
+        if (!f)
+                return ERR_PTR(-EINVAL);
+
+        if (f->f_op == &ion_fops) {
+                client = f->private_data;
+                ion_client_get(client);
+        }
+
+        fput(f);
+        return client;
+}
diff --git a/drivers/gpu/ion/ion_carveout_heap.c b/drivers/gpu/ion/ion_carveout_heap.c
new file mode 100644
index 00000000000..606adae13f4
--- /dev/null
+++ b/drivers/gpu/ion/ion_carveout_heap.c
@@ -0,0 +1,162 @@
+/*
+ * drivers/gpu/ion/ion_carveout_heap.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+#include <linux/spinlock.h>
+
+#include <linux/err.h>
+#include <linux/genalloc.h>
+#include <linux/io.h>
+#include <linux/ion.h>
+#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include "ion_priv.h"
+
+#include <asm/mach/map.h>
+
+struct ion_carveout_heap {
+        struct ion_heap heap;
+        struct gen_pool *pool;
+        ion_phys_addr_t base;
+};
+
+ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap,
+                                      unsigned long size,
+                                      unsigned long align)
+{
+        struct ion_carveout_heap *carveout_heap =
+                container_of(heap, struct ion_carveout_heap, heap);
+        unsigned long offset = gen_pool_alloc(carveout_heap->pool, size);
+
+        if (!offset)
+                return ION_CARVEOUT_ALLOCATE_FAIL;
+
+        return offset;
+}
+
+void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
+                       unsigned long size)
+{
+        struct ion_carveout_heap *carveout_heap =
+                container_of(heap, struct ion_carveout_heap, heap);
+
+        if (addr == ION_CARVEOUT_ALLOCATE_FAIL)
+                return;
+        gen_pool_free(carveout_heap->pool, addr, size);
+}
+
+static int ion_carveout_heap_phys(struct ion_heap *heap,
+                                  struct ion_buffer *buffer,
+                                  ion_phys_addr_t *addr, size_t *len)
+{
+        *addr = buffer->priv_phys;
+        *len = buffer->size;
+        return 0;
+}
+
+static int ion_carveout_heap_allocate(struct ion_heap *heap,
+                                      struct ion_buffer *buffer,
+                                      unsigned long size, unsigned long align,
+                                      unsigned long flags)
+{
+        buffer->priv_phys = ion_carveout_allocate(heap, size, align);
+        return buffer->priv_phys == ION_CARVEOUT_ALLOCATE_FAIL ? -ENOMEM : 0;
+}
+
+static void ion_carveout_heap_free(struct ion_buffer *buffer)
+{
+        struct ion_heap *heap = buffer->heap;
+
+        ion_carveout_free(heap, buffer->priv_phys, buffer->size);
+        buffer->priv_phys = ION_CARVEOUT_ALLOCATE_FAIL;
+}
+
+struct scatterlist *ion_carveout_heap_map_dma(struct ion_heap *heap,
+                                              struct ion_buffer *buffer)
+{
+        return ERR_PTR(-EINVAL);
+}
+
+void ion_carveout_heap_unmap_dma(struct ion_heap *heap,
+                                 struct ion_buffer *buffer)
+{
+        return;
+}
+
+void *ion_carveout_heap_map_kernel(struct ion_heap *heap,
+                                   struct ion_buffer *buffer)
+{
+        return __arch_ioremap(buffer->priv_phys, buffer->size,
+                              MT_MEMORY_NONCACHED);
+}
+
+void ion_carveout_heap_unmap_kernel(struct ion_heap *heap,
+                                    struct ion_buffer *buffer)
+{
+        __arch_iounmap(buffer->vaddr);
+        buffer->vaddr = NULL;
+        return;
+}
+
+int ion_carveout_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer,
+                               struct vm_area_struct *vma)
+{
+        return remap_pfn_range(vma, vma->vm_start,
+                               __phys_to_pfn(buffer->priv_phys) + vma->vm_pgoff,
+                               buffer->size,
+                               pgprot_noncached(vma->vm_page_prot));
+}
+
+static struct ion_heap_ops carveout_heap_ops = {
+        .allocate = ion_carveout_heap_allocate,
+        .free = ion_carveout_heap_free,
+        .phys = ion_carveout_heap_phys,
+        .map_user = ion_carveout_heap_map_user,
+        .map_kernel = ion_carveout_heap_map_kernel,
+        .unmap_kernel = ion_carveout_heap_unmap_kernel,
+};
+
+struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *heap_data)
+{
+        struct ion_carveout_heap *carveout_heap;
+
+        carveout_heap = kzalloc(sizeof(struct ion_carveout_heap), GFP_KERNEL);
+        if (!carveout_heap)
+                return ERR_PTR(-ENOMEM);
+
+        carveout_heap->pool = gen_pool_create(12, -1);
+        if (!carveout_heap->pool) {
+                kfree(carveout_heap);
+                return ERR_PTR(-ENOMEM);
+        }
+        carveout_heap->base = heap_data->base;
+        gen_pool_add(carveout_heap->pool, carveout_heap->base, heap_data->size,
+                     -1);
+        carveout_heap->heap.ops = &carveout_heap_ops;
+        carveout_heap->heap.type = ION_HEAP_TYPE_CARVEOUT;
+
+        return &carveout_heap->heap;
+}
+
+void ion_carveout_heap_destroy(struct ion_heap *heap)
+{
+        struct ion_carveout_heap *carveout_heap =
+             container_of(heap, struct  ion_carveout_heap, heap);
+
+        gen_pool_destroy(carveout_heap->pool);
+        kfree(carveout_heap);
+        carveout_heap = NULL;
+}
diff --git a/drivers/gpu/ion/ion_heap.c b/drivers/gpu/ion/ion_heap.c
new file mode 100644
index 00000000000..6d09778745c
--- /dev/null
+++ b/drivers/gpu/ion/ion_heap.c
@@ -0,0 +1,78 @@
+/*
+ * drivers/gpu/ion/ion_heap.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/ion.h>
+#include "ion_priv.h"
+
+struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data)
+{
+        struct ion_heap *heap = NULL;
+
+        switch (heap_data->type) {
+        case ION_HEAP_TYPE_SYSTEM_CONTIG:
+                heap = ion_system_contig_heap_create(heap_data);
+                break;
+        case ION_HEAP_TYPE_SYSTEM:
+                heap = ion_system_heap_create(heap_data);
+                break;
+        case ION_HEAP_TYPE_CARVEOUT:
+                heap = ion_carveout_heap_create(heap_data);
+                break;
+        case ION_HEAP_TYPE_IOMMU:
+                heap = ion_iommu_heap_create(heap_data);
+                break;
+        default:
+                pr_err("%s: Invalid heap type %d\n", __func__,
+                       heap_data->type);
+                return ERR_PTR(-EINVAL);
+        }
+
+        if (IS_ERR_OR_NULL(heap)) {
+                pr_err("%s: error creating heap %s type %d base %lu size %u\n",
+                       __func__, heap_data->name, heap_data->type,
+                       heap_data->base, heap_data->size);
+                return ERR_PTR(-EINVAL);
+        }
+
+        heap->name = heap_data->name;
+        heap->id = heap_data->id;
+        return heap;
+}
+
+void ion_heap_destroy(struct ion_heap *heap)
+{
+        if (!heap)
+                return;
+
+        switch (heap->type) {
+        case ION_HEAP_TYPE_SYSTEM_CONTIG:
+                ion_system_contig_heap_destroy(heap);
+                break;
+        case ION_HEAP_TYPE_SYSTEM:
+                ion_system_heap_destroy(heap);
+                break;
+        case ION_HEAP_TYPE_CARVEOUT:
+                ion_carveout_heap_destroy(heap);
+                break;
+        case ION_HEAP_TYPE_IOMMU:
+                ion_iommu_heap_destroy(heap);
+                break;
+        default:
+                pr_err("%s: Invalid heap type %d\n", __func__,
+                       heap->type);
+        }
+}
diff --git a/drivers/gpu/ion/ion_iommu_heap.c b/drivers/gpu/ion/ion_iommu_heap.c
new file mode 100644
index 00000000000..a3d2d726bda
--- /dev/null
+++ b/drivers/gpu/ion/ion_iommu_heap.c
@@ -0,0 +1,382 @@
+/*
+ * drivers/gpu/ion/ion_iommu_heap.c
+ *
+ * Copyright (c) 2012, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#define pr_fmt(fmt)     "%s(): " fmt, __func__
+
+#include <linux/spinlock.h>
+#include <linux/kernel.h>
+#include <linux/genalloc.h>
+#include <linux/io.h>
+#include <linux/ion.h>
+#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/iommu.h>
+#include <linux/highmem.h>
+#include <linux/platform_device.h>
+
+#include <asm/cacheflush.h>
+
+#include "ion_priv.h"
+
+#define NUM_PAGES(buf)  (PAGE_ALIGN((buf)->size) >> PAGE_SHIFT)
+
+#define GFP_ION         (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
+
+struct ion_iommu_heap {
+        struct ion_heap         heap;
+        struct gen_pool         *pool;
+        struct iommu_domain     *domain;
+        struct device           *dev;
+};
+
+static struct scatterlist *iommu_heap_map_dma(struct ion_heap *heap,
+                                              struct ion_buffer *buf)
+{
+        struct ion_iommu_heap *h =
+                container_of(heap, struct ion_iommu_heap, heap);
+        int err, npages = NUM_PAGES(buf);
+        unsigned int i;
+        struct scatterlist *sg;
+        unsigned long da = (unsigned long)buf->priv_virt;
+
+        for_each_sg(buf->sglist, sg, npages, i) {
+                phys_addr_t pa;
+
+                pa = sg_phys(sg);
+                BUG_ON(!IS_ALIGNED(sg->length, PAGE_SIZE));
+                err = iommu_map(h->domain, da, pa, PAGE_SIZE, 0);
+                if (err)
+                        goto err_out;
+
+                sg->dma_address = da;
+                da += PAGE_SIZE;
+        }
+
+        pr_debug("da:%p pa:%08x va:%p\n",
+                 buf->priv_virt, sg_phys(buf->sglist), buf->vaddr);
+
+        return buf->sglist;
+
+err_out:
+        if (i-- > 0) {
+                unsigned int j;
+                for_each_sg(buf->sglist, sg, i, j)
+                        iommu_unmap(h->domain, sg_dma_address(sg), 0);
+        }
+        return ERR_PTR(err);
+}
+
+static void iommu_heap_unmap_dma(struct ion_heap *heap, struct ion_buffer *buf)
+{
+        struct ion_iommu_heap *h =
+                container_of(heap, struct ion_iommu_heap, heap);
+        unsigned int i;
+        struct scatterlist *sg;
+        int npages = NUM_PAGES(buf);
+
+        for_each_sg(buf->sglist, sg, npages, i)
+                iommu_unmap(h->domain, sg_dma_address(sg), 0);
+
+        pr_debug("da:%p\n", buf->priv_virt);
+}
+
+struct scatterlist *iommu_heap_remap_dma(struct ion_heap *heap,
+                                              struct ion_buffer *buf,
+                                              unsigned long addr)
+{
+        struct ion_iommu_heap *h =
+                container_of(heap, struct ion_iommu_heap, heap);
+        int err;
+        unsigned int i;
+        unsigned long da, da_to_free = (unsigned long)buf->priv_virt;
+        int npages = NUM_PAGES(buf);
+
+        BUG_ON(!buf->priv_virt);
+
+        da = gen_pool_alloc_addr(h->pool, buf->size, addr);
+        if (da == 0) {
+                pr_err("dma address alloc failed, addr=0x%lx", addr);
+                return ERR_PTR(-ENOMEM);
+        } else {
+                pr_err("iommu_heap_remap_dma passed, addr=0x%lx",
+                        addr);
+                iommu_heap_unmap_dma(heap, buf);
+                gen_pool_free(h->pool, da_to_free, buf->size);
+                buf->priv_virt = (void *)da;
+        }
+        for (i = 0; i < npages; i++) {
+                phys_addr_t pa;
+
+                pa = page_to_phys(buf->pages[i]);
+                err = iommu_map(h->domain, da, pa, 0, 0);
+                if (err)
+                        goto err_out;
+                da += PAGE_SIZE;
+        }
+
+        pr_debug("da:%p pa:%08x va:%p\n",
+                 buf->priv_virt, page_to_phys(buf->pages[0]), buf->vaddr);
+
+        return (struct scatterlist *)buf->pages;
+
+err_out:
+        if (i-- > 0) {
+                da = (unsigned long)buf->priv_virt;
+                iommu_unmap(h->domain, da + (i << PAGE_SHIFT), 0);
+        }
+        return ERR_PTR(err);
+}
+
+static int ion_buffer_allocate(struct ion_buffer *buf)
+{
+        int i, npages = NUM_PAGES(buf);
+
+        buf->pages = kmalloc(npages * sizeof(*buf->pages), GFP_KERNEL);
+        if (!buf->pages)
+                goto err_pages;
+
+        buf->sglist = vzalloc(npages * sizeof(*buf->sglist));
+        if (!buf->sglist)
+                goto err_sgl;
+
+        sg_init_table(buf->sglist, npages);
+
+        for (i = 0; i < npages; i++) {
+                struct page *page;
+                phys_addr_t pa;
+
+                page = alloc_page(GFP_ION);
+                if (!page)
+                        goto err_pgalloc;
+                pa = page_to_phys(page);
+
+                sg_set_page(&buf->sglist[i], page, PAGE_SIZE, 0);
+
+                flush_dcache_page(page);
+                outer_flush_range(pa, pa + PAGE_SIZE);
+
+                buf->pages[i] = page;
+
+                pr_debug_once("pa:%08x\n", pa);
+        }
+        return 0;
+
+err_pgalloc:
+        while (i-- > 0)
+                __free_page(buf->pages[i]);
+        vfree(buf->sglist);
+err_sgl:
+        kfree(buf->pages);
+err_pages:
+        return -ENOMEM;
+}
+
+static void ion_buffer_free(struct ion_buffer *buf)
+{
+        int i, npages = NUM_PAGES(buf);
+
+        for (i = 0; i < npages; i++)
+                __free_page(buf->pages[i]);
+        vfree(buf->sglist);
+        kfree(buf->pages);
+}
+
+static int iommu_heap_allocate(struct ion_heap *heap, struct ion_buffer *buf,
+                               unsigned long len, unsigned long align,
+                               unsigned long flags)
+{
+        int err;
+        struct ion_iommu_heap *h =
+                container_of(heap, struct ion_iommu_heap, heap);
+        unsigned long da;
+        struct scatterlist *sgl;
+
+        len = round_up(len, PAGE_SIZE);
+
+        da = gen_pool_alloc(h->pool, len);
+        if (!da)
+                return -ENOMEM;
+
+        buf->priv_virt = (void *)da;
+        buf->size = len;
+
+        WARN_ON(!IS_ALIGNED(da, PAGE_SIZE));
+
+        err = ion_buffer_allocate(buf);
+        if (err)
+                goto err_alloc_buf;
+
+        sgl = iommu_heap_map_dma(heap, buf);
+        if (IS_ERR_OR_NULL(sgl))
+                goto err_heap_map_dma;
+        buf->vaddr = 0;
+        return 0;
+
+err_heap_map_dma:
+        ion_buffer_free(buf);
+err_alloc_buf:
+        gen_pool_free(h->pool, da, len);
+        buf->size = 0;
+        buf->pages = NULL;
+        buf->priv_virt = NULL;
+        return err;
+}
+
+static void iommu_heap_free(struct ion_buffer *buf)
+{
+        struct ion_heap *heap = buf->heap;
+        struct ion_iommu_heap *h =
+                container_of(heap, struct ion_iommu_heap, heap);
+        void *da = buf->priv_virt;
+
+        iommu_heap_unmap_dma(heap, buf);
+        ion_buffer_free(buf);
+        gen_pool_free(h->pool, (unsigned long)da, buf->size);
+
+        buf->pages = NULL;
+        buf->priv_virt = NULL;
+        pr_debug("da:%p\n", da);
+}
+
+static int iommu_heap_phys(struct ion_heap *heap, struct ion_buffer *buf,
+                           ion_phys_addr_t *addr, size_t *len)
+{
+        *addr = (unsigned long)buf->priv_virt;
+        *len = buf->size;
+        pr_debug("da:%08lx(%x)\n", *addr, *len);
+        return 0;
+}
+
+static void *iommu_heap_map_kernel(struct ion_heap *heap,
+                                   struct ion_buffer *buf)
+{
+        int npages = NUM_PAGES(buf);
+
+        BUG_ON(!buf->pages);
+        buf->vaddr = vm_map_ram(buf->pages, npages, -1,
+                                pgprot_noncached(pgprot_kernel));
+        pr_debug("va:%p\n", buf->vaddr);
+        WARN_ON(!buf->vaddr);
+        return buf->vaddr;
+}
+
+static void iommu_heap_unmap_kernel(struct ion_heap *heap,
+                                    struct ion_buffer *buf)
+{
+        int npages = NUM_PAGES(buf);
+
+        BUG_ON(!buf->pages);
+        WARN_ON(!buf->vaddr);
+        vm_unmap_ram(buf->vaddr, npages);
+        buf->vaddr = NULL;
+        pr_debug("va:%p\n", buf->vaddr);
+}
+
+static int iommu_heap_map_user(struct ion_heap *mapper,
+                               struct ion_buffer *buf,
+                               struct vm_area_struct *vma)
+{
+        int i = vma->vm_pgoff >> PAGE_SHIFT;
+        unsigned long uaddr = vma->vm_start;
+        unsigned long usize = vma->vm_end - vma->vm_start;
+
+        pr_debug("vma:%08lx-%08lx\n", vma->vm_start, vma->vm_end);
+        BUG_ON(!buf->pages);
+
+        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+        do {
+                int ret;
+                struct page *page = buf->pages[i++];
+
+                ret = vm_insert_page(vma, uaddr, page);
+                if (ret)
+                        return ret;
+
+                uaddr += PAGE_SIZE;
+                usize -= PAGE_SIZE;
+        } while (usize > 0);
+
+        return 0;
+}
+
+static struct ion_heap_ops iommu_heap_ops = {
+        .allocate       = iommu_heap_allocate,
+        .free           = iommu_heap_free,
+        .phys           = iommu_heap_phys,
+        .map_dma        = iommu_heap_map_dma,
+        .unmap_dma      = iommu_heap_unmap_dma,
+        .map_kernel     = iommu_heap_map_kernel,
+        .unmap_kernel   = iommu_heap_unmap_kernel,
+        .map_user       = iommu_heap_map_user,
+};
+
+struct ion_heap *ion_iommu_heap_create(struct ion_platform_heap *data)
+{
+        struct ion_iommu_heap *h;
+        int err;
+
+        h = kzalloc(sizeof(*h), GFP_KERNEL);
+        if (!h) {
+                err = -ENOMEM;
+                goto err_heap;
+        }
+
+        h->pool = gen_pool_create(12, -1);
+        if (!h->pool) {
+                err = -ENOMEM;
+                goto err_genpool;
+        }
+        gen_pool_add(h->pool, data->base, data->size, -1);
+
+        h->heap.ops = &iommu_heap_ops;
+        h->domain = iommu_domain_alloc(&platform_bus_type);
+        h->dev = data->priv;
+        if (!h->domain) {
+                err = -ENOMEM;
+                goto err_iommu_alloc;
+        }
+
+        err = iommu_attach_device(h->domain, h->dev);
+        if (err)
+                goto err_iommu_attach;
+
+        return &h->heap;
+
+err_iommu_attach:
+        iommu_domain_free(h->domain);
+err_iommu_alloc:
+        gen_pool_destroy(h->pool);
+err_genpool:
+        kfree(h);
+err_heap:
+        return ERR_PTR(err);
+}
+
+void ion_iommu_heap_destroy(struct ion_heap *heap)
+{
+        struct ion_iommu_heap *h =
+                container_of(heap, struct  ion_iommu_heap, heap);
+
+        iommu_detach_device(h->domain, h->dev);
+        gen_pool_destroy(h->pool);
+        iommu_domain_free(h->domain);
+        kfree(h);
+}
diff --git a/drivers/gpu/ion/ion_priv.h b/drivers/gpu/ion/ion_priv.h
new file mode 100644
index 00000000000..bfe26da9c04
--- /dev/null
+++ b/drivers/gpu/ion/ion_priv.h
@@ -0,0 +1,293 @@
+/*
+ * drivers/gpu/ion/ion_priv.h
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _ION_PRIV_H
+#define _ION_PRIV_H
+
+#include <linux/kref.h>
+#include <linux/mm_types.h>
+#include <linux/mutex.h>
+#include <linux/rbtree.h>
+#include <linux/ion.h>
+#include <linux/miscdevice.h>
+
+struct ion_mapping;
+
+struct ion_dma_mapping {
+        struct kref ref;
+        struct scatterlist *sglist;
+};
+
+struct ion_kernel_mapping {
+        struct kref ref;
+        void *vaddr;
+};
+
+/**
+ * struct ion_device - the metadata of the ion device node
+ * @dev:                the actual misc device
+ * @buffers:    an rb tree of all the existing buffers
+ * @lock:               lock protecting the buffers & heaps trees
+ * @heaps:              list of all the heaps in the system
+ * @user_clients:       list of all the clients created from userspace
+ */
+struct ion_device {
+        struct miscdevice dev;
+        struct rb_root buffers;
+        struct mutex lock;
+        struct rb_root heaps;
+        long (*custom_ioctl) (struct ion_client *client, unsigned int cmd,
+                              unsigned long arg);
+        struct rb_root user_clients;
+        struct rb_root kernel_clients;
+        struct dentry *debug_root;
+};
+
+/**
+ * struct ion_client - a process/hw block local address space
+ * @ref:                for reference counting the client
+ * @node:               node in the tree of all clients
+ * @dev:                backpointer to ion device
+ * @handles:            an rb tree of all the handles in this client
+ * @lock:               lock protecting the tree of handles
+ * @heap_mask:          mask of all supported heaps
+ * @name:               used for debugging
+ * @task:               used for debugging
+ *
+ * A client represents a list of buffers this client may access.
+ * The mutex stored here is used to protect both handles tree
+ * as well as the handles themselves, and should be held while modifying either.
+ */
+struct ion_client {
+        struct kref ref;
+        struct rb_node node;
+        struct ion_device *dev;
+        struct rb_root handles;
+        struct mutex lock;
+        unsigned int heap_mask;
+        const char *name;
+        struct task_struct *task;
+        pid_t pid;
+        struct dentry *debug_root;
+};
+
+/**
+ * ion_handle - a client local reference to a buffer
+ * @ref:                reference count
+ * @client:             back pointer to the client the buffer resides in
+ * @buffer:             pointer to the buffer
+ * @node:               node in the client's handle rbtree
+ * @kmap_cnt:           count of times this client has mapped to kernel
+ * @dmap_cnt:           count of times this client has mapped for dma
+ * @usermap_cnt:        count of times this client has mapped for userspace
+ *
+ * Modifications to node, map_cnt or mapping should be protected by the
+ * lock in the client.  Other fields are never changed after initialization.
+ */
+struct ion_handle {
+        struct kref ref;
+        struct ion_client *client;
+        struct ion_buffer *buffer;
+        struct rb_node node;
+        unsigned int kmap_cnt;
+        unsigned int dmap_cnt;
+        unsigned int usermap_cnt;
+};
+
+bool ion_handle_validate(struct ion_client *client, struct ion_handle *handle);
+
+void ion_buffer_get(struct ion_buffer *buffer);
+
+struct ion_buffer *ion_handle_buffer(struct ion_handle *handle);
+
+struct ion_client *ion_client_get_file(int fd);
+
+void ion_client_get(struct ion_client *client);
+
+int ion_client_put(struct ion_client *client);
+
+void ion_handle_get(struct ion_handle *handle);
+
+int ion_handle_put(struct ion_handle *handle);
+
+struct ion_handle *ion_handle_create(struct ion_client *client,
+                                     struct ion_buffer *buffer);
+
+void ion_handle_add(struct ion_client *client, struct ion_handle *handle);
+
+int ion_remap_dma(struct ion_client *client,
+                    struct ion_handle *handle,
+                    unsigned long addr);
+/**
+ * struct ion_buffer - metadata for a particular buffer
+ * @ref:                refernce count
+ * @node:               node in the ion_device buffers tree
+ * @dev:                back pointer to the ion_device
+ * @heap:               back pointer to the heap the buffer came from
+ * @flags:              buffer specific flags
+ * @size:               size of the buffer
+ * @priv_virt:          private data to the buffer representable as
+ *                      a void *
+ * @priv_phys:          private data to the buffer representable as
+ *                      an ion_phys_addr_t (and someday a phys_addr_t)
+ * @lock:               protects the buffers cnt fields
+ * @kmap_cnt:           number of times the buffer is mapped to the kernel
+ * @vaddr:              the kenrel mapping if kmap_cnt is not zero
+ * @dmap_cnt:           number of times the buffer is mapped for dma
+ * @sglist:             the scatterlist for the buffer is dmap_cnt is not zero
+ * @pages:              list for allocated pages for the buffer
+ */
+struct ion_buffer {
+        struct kref ref;
+        struct rb_node node;
+        struct ion_device *dev;
+        struct ion_heap *heap;
+        unsigned long flags;
+        size_t size;
+        union {
+                void *priv_virt;
+                ion_phys_addr_t priv_phys;
+        };
+        struct mutex lock;
+        int kmap_cnt;
+        void *vaddr;
+        int dmap_cnt;
+        struct scatterlist *sglist;
+        struct page **pages;
+};
+
+/**
+ * struct ion_heap_ops - ops to operate on a given heap
+ * @allocate:           allocate memory
+ * @free:               free memory
+ * @phys                get physical address of a buffer (only define on
+ *                      physically contiguous heaps)
+ * @map_dma             map the memory for dma to a scatterlist
+ * @unmap_dma           unmap the memory for dma
+ * @map_kernel          map memory to the kernel
+ * @unmap_kernel        unmap memory to the kernel
+ * @map_user            map memory to userspace
+ */
+struct ion_heap_ops {
+        int (*allocate) (struct ion_heap *heap,
+                         struct ion_buffer *buffer, unsigned long len,
+                         unsigned long align, unsigned long flags);
+        void (*free) (struct ion_buffer *buffer);
+        int (*phys) (struct ion_heap *heap, struct ion_buffer *buffer,
+                     ion_phys_addr_t *addr, size_t *len);
+        struct scatterlist *(*map_dma) (struct ion_heap *heap,
+                                        struct ion_buffer *buffer);
+        void (*unmap_dma) (struct ion_heap *heap, struct ion_buffer *buffer);
+        void * (*map_kernel) (struct ion_heap *heap, struct ion_buffer *buffer);
+        void (*unmap_kernel) (struct ion_heap *heap, struct ion_buffer *buffer);
+        int (*map_user) (struct ion_heap *mapper, struct ion_buffer *buffer,
+                         struct vm_area_struct *vma);
+};
+
+/**
+ * struct ion_heap - represents a heap in the system
+ * @node:               rb node to put the heap on the device's tree of heaps
+ * @dev:                back pointer to the ion_device
+ * @type:               type of heap
+ * @ops:                ops struct as above
+ * @id:                 id of heap, also indicates priority of this heap when
+ *                      allocating.  These are specified by platform data and
+ *                      MUST be unique
+ * @name:               used for debugging
+ *
+ * Represents a pool of memory from which buffers can be made.  In some
+ * systems the only heap is regular system memory allocated via vmalloc.
+ * On others, some blocks might require large physically contiguous buffers
+ * that are allocated from a specially reserved heap.
+ */
+struct ion_heap {
+        struct rb_node node;
+        struct ion_device *dev;
+        enum ion_heap_type type;
+        struct ion_heap_ops *ops;
+        int id;
+        const char *name;
+};
+
+/**
+ * ion_device_create - allocates and returns an ion device
+ * @custom_ioctl:       arch specific ioctl function if applicable
+ *
+ * returns a valid device or -PTR_ERR
+ */
+struct ion_device *ion_device_create(long (*custom_ioctl)
+                                     (struct ion_client *client,
+                                      unsigned int cmd,
+                                      unsigned long arg));
+
+/**
+ * ion_device_destroy - free and device and it's resource
+ * @dev:                the device
+ */
+void ion_device_destroy(struct ion_device *dev);
+
+/**
+ * ion_device_add_heap - adds a heap to the ion device
+ * @dev:                the device
+ * @heap:               the heap to add
+ */
+void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap);
+
+/**
+ * functions for creating and destroying the built in ion heaps.
+ * architectures can add their own custom architecture specific
+ * heaps as appropriate.
+ */
+
+struct ion_heap *ion_heap_create(struct ion_platform_heap *);
+void ion_heap_destroy(struct ion_heap *);
+
+struct ion_heap *ion_system_heap_create(struct ion_platform_heap *);
+void ion_system_heap_destroy(struct ion_heap *);
+
+struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *);
+void ion_system_contig_heap_destroy(struct ion_heap *);
+
+struct ion_heap *ion_carveout_heap_create(struct ion_platform_heap *);
+void ion_carveout_heap_destroy(struct ion_heap *);
+/**
+ * kernel api to allocate/free from carveout -- used when carveout is
+ * used to back an architecture specific custom heap
+ */
+ion_phys_addr_t ion_carveout_allocate(struct ion_heap *heap, unsigned long size,
+                                      unsigned long align);
+void ion_carveout_free(struct ion_heap *heap, ion_phys_addr_t addr,
+                       unsigned long size);
+
+#ifdef CONFIG_ION_IOMMU
+struct ion_heap *ion_iommu_heap_create(struct ion_platform_heap *);
+void ion_iommu_heap_destroy(struct ion_heap *);
+#else
+static inline struct ion_heap *ion_iommu_heap_create(struct ion_platform_heap *)
+{
+        return NULL;
+}
+static inline void ion_iommu_heap_destroy(struct ion_heap *)
+{
+}
+#endif
+/**
+ * The carveout heap returns physical addresses, since 0 may be a valid
+ * physical address, this is used to indicate allocation failed
+ */
+#define ION_CARVEOUT_ALLOCATE_FAIL -1
+
+#endif /* _ION_PRIV_H */
diff --git a/drivers/gpu/ion/ion_system_heap.c b/drivers/gpu/ion/ion_system_heap.c
new file mode 100644
index 00000000000..c046cf1a321
--- /dev/null
+++ b/drivers/gpu/ion/ion_system_heap.c
@@ -0,0 +1,198 @@
+/*
+ * drivers/gpu/ion/ion_system_heap.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/ion.h>
+#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include "ion_priv.h"
+
+static int ion_system_heap_allocate(struct ion_heap *heap,
+                                     struct ion_buffer *buffer,
+                                     unsigned long size, unsigned long align,
+                                     unsigned long flags)
+{
+        buffer->priv_virt = vmalloc_user(size);
+        if (!buffer->priv_virt)
+                return -ENOMEM;
+        return 0;
+}
+
+void ion_system_heap_free(struct ion_buffer *buffer)
+{
+        vfree(buffer->priv_virt);
+}
+
+struct scatterlist *ion_system_heap_map_dma(struct ion_heap *heap,
+                                            struct ion_buffer *buffer)
+{
+        struct scatterlist *sglist;
+        struct page *page;
+        int i;
+        int npages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
+        void *vaddr = buffer->priv_virt;
+
+        sglist = vmalloc(npages * sizeof(struct scatterlist));
+        if (!sglist)
+                return ERR_PTR(-ENOMEM);
+        memset(sglist, 0, npages * sizeof(struct scatterlist));
+        sg_init_table(sglist, npages);
+        for (i = 0; i < npages; i++) {
+                page = vmalloc_to_page(vaddr);
+                if (!page)
+                        goto end;
+                sg_set_page(&sglist[i], page, PAGE_SIZE, 0);
+                vaddr += PAGE_SIZE;
+        }
+        /* XXX do cache maintenance for dma? */
+        return sglist;
+end:
+        vfree(sglist);
+        return NULL;
+}
+
+void ion_system_heap_unmap_dma(struct ion_heap *heap,
+                               struct ion_buffer *buffer)
+{
+        /* XXX undo cache maintenance for dma? */
+        if (buffer->sglist)
+                vfree(buffer->sglist);
+}
+
+void *ion_system_heap_map_kernel(struct ion_heap *heap,
+                                 struct ion_buffer *buffer)
+{
+        return buffer->priv_virt;
+}
+
+void ion_system_heap_unmap_kernel(struct ion_heap *heap,
+                                  struct ion_buffer *buffer)
+{
+}
+
+int ion_system_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer,
+                             struct vm_area_struct *vma)
+{
+        return remap_vmalloc_range(vma, buffer->priv_virt, vma->vm_pgoff);
+}
+
+static struct ion_heap_ops vmalloc_ops = {
+        .allocate = ion_system_heap_allocate,
+        .free = ion_system_heap_free,
+        .map_dma = ion_system_heap_map_dma,
+        .unmap_dma = ion_system_heap_unmap_dma,
+        .map_kernel = ion_system_heap_map_kernel,
+        .unmap_kernel = ion_system_heap_unmap_kernel,
+        .map_user = ion_system_heap_map_user,
+};
+
+struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused)
+{
+        struct ion_heap *heap;
+
+        heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL);
+        if (!heap)
+                return ERR_PTR(-ENOMEM);
+        heap->ops = &vmalloc_ops;
+        heap->type = ION_HEAP_TYPE_SYSTEM;
+        return heap;
+}
+
+void ion_system_heap_destroy(struct ion_heap *heap)
+{
+        kfree(heap);
+}
+
+static int ion_system_contig_heap_allocate(struct ion_heap *heap,
+                                           struct ion_buffer *buffer,
+                                           unsigned long len,
+                                           unsigned long align,
+                                           unsigned long flags)
+{
+        buffer->priv_virt = kzalloc(len, GFP_KERNEL);
+        if (!buffer->priv_virt)
+                return -ENOMEM;
+        return 0;
+}
+
+void ion_system_contig_heap_free(struct ion_buffer *buffer)
+{
+        kfree(buffer->priv_virt);
+}
+
+static int ion_system_contig_heap_phys(struct ion_heap *heap,
+                                       struct ion_buffer *buffer,
+                                       ion_phys_addr_t *addr, size_t *len)
+{
+        *addr = virt_to_phys(buffer->priv_virt);
+        *len = buffer->size;
+        return 0;
+}
+
+struct scatterlist *ion_system_contig_heap_map_dma(struct ion_heap *heap,
+                                                   struct ion_buffer *buffer)
+{
+        struct scatterlist *sglist;
+
+        sglist = vmalloc(sizeof(struct scatterlist));
+        if (!sglist)
+                return ERR_PTR(-ENOMEM);
+        sg_init_table(sglist, 1);
+        sg_set_page(sglist, virt_to_page(buffer->priv_virt), buffer->size, 0);
+        return sglist;
+}
+
+int ion_system_contig_heap_map_user(struct ion_heap *heap,
+                                    struct ion_buffer *buffer,
+                                    struct vm_area_struct *vma)
+{
+        unsigned long pfn = __phys_to_pfn(virt_to_phys(buffer->priv_virt));
+        return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
+                               vma->vm_end - vma->vm_start,
+                               vma->vm_page_prot);
+
+}
+
+static struct ion_heap_ops kmalloc_ops = {
+        .allocate = ion_system_contig_heap_allocate,
+        .free = ion_system_contig_heap_free,
+        .phys = ion_system_contig_heap_phys,
+        .map_dma = ion_system_contig_heap_map_dma,
+        .unmap_dma = ion_system_heap_unmap_dma,
+        .map_kernel = ion_system_heap_map_kernel,
+        .unmap_kernel = ion_system_heap_unmap_kernel,
+        .map_user = ion_system_contig_heap_map_user,
+};
+
+struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused)
+{
+        struct ion_heap *heap;
+
+        heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL);
+        if (!heap)
+                return ERR_PTR(-ENOMEM);
+        heap->ops = &kmalloc_ops;
+        heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG;
+        return heap;
+}
+
+void ion_system_contig_heap_destroy(struct ion_heap *heap)
+{
+        kfree(heap);
+}
+
diff --git a/drivers/gpu/ion/ion_system_mapper.c b/drivers/gpu/ion/ion_system_mapper.c
new file mode 100644
index 00000000000..692458e07b5
--- /dev/null
+++ b/drivers/gpu/ion/ion_system_mapper.c
@@ -0,0 +1,114 @@
+/*
+ * drivers/gpu/ion/ion_system_mapper.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/ion.h>
+#include <linux/memory.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include "ion_priv.h"
+/*
+ * This mapper is valid for any heap that allocates memory that already has
+ * a kernel mapping, this includes vmalloc'd memory, kmalloc'd memory,
+ * pages obtained via io_remap, etc.
+ */
+static void *ion_kernel_mapper_map(struct ion_mapper *mapper,
+                                   struct ion_buffer *buffer,
+                                   struct ion_mapping **mapping)
+{
+        if (!((1 << buffer->heap->type) & mapper->heap_mask)) {
+                pr_err("%s: attempting to map an unsupported heap\n", __func__);
+                return ERR_PTR(-EINVAL);
+        }
+        /* XXX REVISIT ME!!! */
+        *((unsigned long *)mapping) = (unsigned long)buffer->priv;
+        return buffer->priv;
+}
+
+static void ion_kernel_mapper_unmap(struct ion_mapper *mapper,
+                                    struct ion_buffer *buffer,
+                                    struct ion_mapping *mapping)
+{
+        if (!((1 << buffer->heap->type) & mapper->heap_mask))
+                pr_err("%s: attempting to unmap an unsupported heap\n",
+                       __func__);
+}
+
+static void *ion_kernel_mapper_map_kernel(struct ion_mapper *mapper,
+                                        struct ion_buffer *buffer,
+                                        struct ion_mapping *mapping)
+{
+        if (!((1 << buffer->heap->type) & mapper->heap_mask)) {
+                pr_err("%s: attempting to unmap an unsupported heap\n",
+                       __func__);
+                return ERR_PTR(-EINVAL);
+        }
+        return buffer->priv;
+}
+
+static int ion_kernel_mapper_map_user(struct ion_mapper *mapper,
+                                      struct ion_buffer *buffer,
+                                      struct vm_area_struct *vma,
+                                      struct ion_mapping *mapping)
+{
+        int ret;
+
+        switch (buffer->heap->type) {
+        case ION_HEAP_KMALLOC:
+        {
+                unsigned long pfn = __phys_to_pfn(virt_to_phys(buffer->priv));
+                ret = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
+                                      vma->vm_end - vma->vm_start,
+                                      vma->vm_page_prot);
+                break;
+        }
+        case ION_HEAP_VMALLOC:
+                ret = remap_vmalloc_range(vma, buffer->priv, vma->vm_pgoff);
+                break;
+        default:
+                pr_err("%s: attempting to map unsupported heap to userspace\n",
+                       __func__);
+                return -EINVAL;
+        }
+
+        return ret;
+}
+
+static struct ion_mapper_ops ops = {
+        .map = ion_kernel_mapper_map,
+        .map_kernel = ion_kernel_mapper_map_kernel,
+        .map_user = ion_kernel_mapper_map_user,
+        .unmap = ion_kernel_mapper_unmap,
+};
+
+struct ion_mapper *ion_system_mapper_create(void)
+{
+        struct ion_mapper *mapper;
+        mapper = kzalloc(sizeof(struct ion_mapper), GFP_KERNEL);
+        if (!mapper)
+                return ERR_PTR(-ENOMEM);
+        mapper->type = ION_SYSTEM_MAPPER;
+        mapper->ops = &ops;
+        mapper->heap_mask = (1 << ION_HEAP_VMALLOC) | (1 << ION_HEAP_KMALLOC);
+        return mapper;
+}
+
+void ion_system_mapper_destroy(struct ion_mapper *mapper)
+{
+        kfree(mapper);
+}
+
diff --git a/drivers/gpu/ion/tegra/Makefile b/drivers/gpu/ion/tegra/Makefile
new file mode 100644
index 00000000000..11cd003fb08
--- /dev/null
+++ b/drivers/gpu/ion/tegra/Makefile
@@ -0,0 +1 @@
+obj-y += tegra_ion.o
diff --git a/drivers/gpu/ion/tegra/tegra_ion.c b/drivers/gpu/ion/tegra/tegra_ion.c
new file mode 100644
index 00000000000..2252079279e
--- /dev/null
+++ b/drivers/gpu/ion/tegra/tegra_ion.c
@@ -0,0 +1,599 @@
+/*
+ * drivers/gpu/tegra/tegra_ion.c
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2011, NVIDIA Corporation.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt)     "%s():%d: " fmt, __func__, __LINE__
+
+#include <linux/err.h>
+#include <linux/ion.h>
+#include <linux/tegra_ion.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/syscalls.h>
+#include <linux/io.h>
+#include "../ion_priv.h"
+
+#define CLIENT_HEAP_MASK 0xFFFFFFFF
+#define HEAP_FLAGS 0xFF
+
+#if !defined(CONFIG_TEGRA_NVMAP)
+#include "mach/nvmap.h"
+struct nvmap_device *nvmap_dev;
+#endif
+
+static struct ion_device *idev;
+static int num_heaps;
+static struct ion_heap **heaps;
+
+static int tegra_ion_pin(struct ion_client *client,
+                                unsigned int cmd,
+                                unsigned long arg)
+{
+        struct tegra_ion_pin_data data;
+        int ret;
+        struct ion_handle *on_stack[16];
+        struct ion_handle **refs = on_stack;
+        int i;
+        bool valid_handle;
+
+        if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                return -EFAULT;
+        if (data.count) {
+                size_t bytes = data.count * sizeof(struct ion_handle *);
+
+                if (data.count > ARRAY_SIZE(on_stack))
+                        refs = kmalloc(data.count * sizeof(*refs), GFP_KERNEL);
+                else
+                        refs = on_stack;
+                if (!refs)
+                        return -ENOMEM;
+                if (copy_from_user(refs, (void *)data.handles, bytes)) {
+                        ret = -EFAULT;
+                        goto err;
+                }
+        } else
+                return -EINVAL;
+
+        mutex_lock(&client->lock);
+        for (i = 0; i < data.count; i++) {
+                /* Ignore NULL pointers during unpin operation. */
+                if (!refs[i] && cmd == TEGRA_ION_UNPIN)
+                        continue;
+                valid_handle = ion_handle_validate(client, refs[i]);
+                if (!valid_handle) {
+                        WARN(1, "invalid handle passed h=0x%x", (u32)refs[i]);
+                        mutex_unlock(&client->lock);
+                        ret = -EINVAL;
+                        goto err;
+                }
+        }
+        mutex_unlock(&client->lock);
+
+        if (cmd == TEGRA_ION_PIN) {
+                ion_phys_addr_t addr;
+                size_t len;
+
+                for (i = 0; i < data.count; i++) {
+                        ret = ion_phys(client, refs[i], &addr, &len);
+                        if (ret)
+                                goto err;
+                        ion_handle_get(refs[i]);
+                        ret = put_user(addr, &data.addr[i]);
+                        if (ret)
+                                return ret;
+                }
+        } else if (cmd == TEGRA_ION_UNPIN) {
+                for (i = 0; i < data.count; i++) {
+                        if (refs[i])
+                                ion_handle_put(refs[i]);
+                }
+        }
+
+err:
+        if (ret) {
+                pr_err("error, ret=0x%x", ret);
+                /* FIXME: undo pinning. */
+        }
+        if (refs != on_stack)
+                kfree(refs);
+        return ret;
+}
+
+static int tegra_ion_alloc_from_id(struct ion_client *client,
+                                    unsigned int cmd,
+                                    unsigned long arg)
+{
+        struct tegra_ion_id_data data;
+        struct ion_buffer *buffer;
+        struct tegra_ion_id_data *user_data = (struct tegra_ion_id_data *)arg;
+
+        if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                return -EFAULT;
+        buffer = (struct ion_buffer *)data.id;
+        data.handle = ion_import(client, buffer);
+        data.size = buffer->size;
+        if (put_user(data.handle, &user_data->handle))
+                return -EFAULT;
+        if (put_user(data.size, &user_data->size))
+                return -EFAULT;
+        return 0;
+}
+
+static int tegra_ion_get_id(struct ion_client *client,
+                                            unsigned int cmd,
+                                            unsigned long arg)
+{
+        bool valid_handle;
+        struct tegra_ion_id_data data;
+        struct tegra_ion_id_data *user_data = (struct tegra_ion_id_data *)arg;
+
+        if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                return -EFAULT;
+
+        mutex_lock(&client->lock);
+        valid_handle = ion_handle_validate(client, data.handle);
+        mutex_unlock(&client->lock);
+
+        if (!valid_handle) {
+                WARN(1, "invalid handle passed\n");
+                return -EINVAL;
+        }
+
+        pr_debug("h=0x%x, b=0x%x, bref=%d",
+                (u32)data.handle, (u32)data.handle->buffer,
+                atomic_read(&data.handle->buffer->ref.refcount));
+        if (put_user((unsigned long)ion_handle_buffer(data.handle),
+                        &user_data->id))
+                return -EFAULT;
+        return 0;
+}
+
+static int tegra_ion_cache_maint(struct ion_client *client,
+                                 unsigned int cmd,
+                                 unsigned long arg)
+{
+        wmb();
+        return 0;
+}
+
+static int tegra_ion_rw(struct ion_client *client,
+                                unsigned int cmd,
+                                unsigned long arg)
+{
+        bool valid_handle;
+        struct tegra_ion_rw_data data;
+        char *kern_addr, *src;
+        int ret = 0;
+        size_t copied = 0;
+
+        if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                return -EFAULT;
+
+        if (!data.handle || !data.addr || !data.count || !data.elem_size)
+                return -EINVAL;
+
+        mutex_lock(&client->lock);
+        valid_handle = ion_handle_validate(client, data.handle);
+        mutex_unlock(&client->lock);
+
+        if (!valid_handle) {
+                WARN(1, "%s: invalid handle passed to get id.\n", __func__);
+                return -EINVAL;
+        }
+
+        if (data.elem_size == data.mem_stride &&
+            data.elem_size == data.user_stride) {
+                data.elem_size *= data.count;
+                data.mem_stride = data.elem_size;
+                data.user_stride = data.elem_size;
+                data.count = 1;
+        }
+
+        kern_addr = ion_map_kernel(client, data.handle);
+
+        while (data.count--) {
+                if (data.offset + data.elem_size > data.handle->buffer->size) {
+                        WARN(1, "read/write outside of handle\n");
+                        ret = -EFAULT;
+                        break;
+                }
+
+                src = kern_addr + data.offset;
+                if (cmd == TEGRA_ION_READ)
+                        ret = copy_to_user((void *)data.addr,
+                                            src, data.elem_size);
+                else
+                        ret = copy_from_user(src,
+                                            (void *)data.addr, data.elem_size);
+
+                if (ret)
+                        break;
+
+                copied += data.elem_size;
+                data.addr += data.user_stride;
+                data.offset += data.mem_stride;
+        }
+
+        ion_unmap_kernel(client, data.handle);
+        return ret;
+}
+
+static int tegra_ion_get_param(struct ion_client *client,
+                                        unsigned int cmd,
+                                        unsigned long arg)
+{
+        bool valid_handle;
+        struct tegra_ion_get_params_data data;
+        struct tegra_ion_get_params_data *user_data =
+                                (struct tegra_ion_get_params_data *)arg;
+        struct ion_buffer *buffer;
+
+        if (copy_from_user(&data, (void __user *)arg, sizeof(data)))
+                return -EFAULT;
+
+        mutex_lock(&client->lock);
+        valid_handle = ion_handle_validate(client, data.handle);
+        mutex_unlock(&client->lock);
+
+        if (!valid_handle) {
+                WARN(1, "%s: invalid handle passed to get id.\n", __func__);
+                return -EINVAL;
+        }
+
+        buffer = ion_handle_buffer(data.handle);
+        data.align = 4096;
+        data.heap = 1;
+        ion_phys(client, data.handle, &data.addr, &data.size);
+
+        if (copy_to_user(user_data, &data, sizeof(data)))
+                return -EFAULT;
+
+        return 0;
+}
+
+static long tegra_ion_ioctl(struct ion_client *client,
+                                   unsigned int cmd,
+                                   unsigned long arg)
+{
+        int ret = -ENOTTY;
+
+        switch (cmd) {
+        case TEGRA_ION_ALLOC_FROM_ID:
+                ret = tegra_ion_alloc_from_id(client, cmd, arg);
+                break;
+        case TEGRA_ION_GET_ID:
+                ret = tegra_ion_get_id(client, cmd, arg);
+                break;
+        case TEGRA_ION_PIN:
+        case TEGRA_ION_UNPIN:
+                ret = tegra_ion_pin(client, cmd, arg);
+                break;
+        case TEGRA_ION_CACHE_MAINT:
+                ret = tegra_ion_cache_maint(client, cmd, arg);
+                break;
+        case TEGRA_ION_READ:
+        case TEGRA_ION_WRITE:
+                ret = tegra_ion_rw(client, cmd, arg);
+                break;
+        case TEGRA_ION_GET_PARAM:
+                ret = tegra_ion_get_param(client, cmd, arg);
+                break;
+        default:
+                WARN(1, "Unknown custom ioctl\n");
+                return -ENOTTY;
+        }
+        return ret;
+}
+
+int tegra_ion_probe(struct platform_device *pdev)
+{
+        struct ion_platform_data *pdata = pdev->dev.platform_data;
+        int i;
+
+        num_heaps = pdata->nr;
+
+        heaps = kzalloc(sizeof(struct ion_heap *) * pdata->nr, GFP_KERNEL);
+
+        idev = ion_device_create(tegra_ion_ioctl);
+        if (IS_ERR_OR_NULL(idev)) {
+                kfree(heaps);
+                return PTR_ERR(idev);
+        }
+
+        /* create the heaps as specified in the board file */
+        for (i = 0; i < num_heaps; i++) {
+                struct ion_platform_heap *heap_data = &pdata->heaps[i];
+
+                heaps[i] = ion_heap_create(heap_data);
+                if (IS_ERR_OR_NULL(heaps[i])) {
+                        pr_warn("%s(type:%d id:%d) isn't supported\n",
+                                heap_data->name,
+                                heap_data->type, heap_data->id);
+                        continue;
+                }
+                ion_device_add_heap(idev, heaps[i]);
+        }
+        platform_set_drvdata(pdev, idev);
+#if !defined(CONFIG_TEGRA_NVMAP)
+        nvmap_dev = (struct nvmap_device *)idev;
+#endif
+        return 0;
+}
+
+int tegra_ion_remove(struct platform_device *pdev)
+{
+        struct ion_device *idev = platform_get_drvdata(pdev);
+        int i;
+
+        ion_device_destroy(idev);
+        for (i = 0; i < num_heaps; i++)
+                ion_heap_destroy(heaps[i]);
+        kfree(heaps);
+        return 0;
+}
+
+static struct platform_driver ion_driver = {
+        .probe = tegra_ion_probe,
+        .remove = tegra_ion_remove,
+        .driver = { .name = "ion-tegra" }
+};
+
+static int __init ion_init(void)
+{
+        return platform_driver_register(&ion_driver);
+}
+
+static void __exit ion_exit(void)
+{
+        platform_driver_unregister(&ion_driver);
+}
+
+fs_initcall(ion_init);
+module_exit(ion_exit);
+
+#if !defined(CONFIG_TEGRA_NVMAP)
+struct nvmap_client *nvmap_create_client(struct nvmap_device *dev,
+                                         const char *name)
+{
+        return ion_client_create(dev, CLIENT_HEAP_MASK, name);
+}
+
+struct nvmap_handle_ref *nvmap_alloc(struct nvmap_client *client, size_t size,
+                                     size_t align, unsigned int flags,
+                                     unsigned int heap_mask)
+{
+        return ion_alloc(client, size, align, HEAP_FLAGS);
+}
+
+void nvmap_free(struct nvmap_client *client, struct nvmap_handle_ref *r)
+{
+        ion_free(client, r);
+}
+
+void *nvmap_mmap(struct nvmap_handle_ref *r)
+{
+        return ion_map_kernel(r->client, r);
+}
+
+void nvmap_munmap(struct nvmap_handle_ref *r, void *addr)
+{
+        ion_unmap_kernel(r->client, r);
+}
+
+struct nvmap_client *nvmap_client_get_file(int fd)
+{
+        return ion_client_get_file(fd);
+}
+
+struct nvmap_client *nvmap_client_get(struct nvmap_client *client)
+{
+        ion_client_get(client);
+        return client;
+}
+
+void nvmap_client_put(struct nvmap_client *c)
+{
+        ion_client_put(c);
+}
+
+phys_addr_t nvmap_pin(struct nvmap_client *c, struct nvmap_handle_ref *r)
+{
+        ion_phys_addr_t addr;
+        size_t len;
+
+        ion_handle_get(r);
+        ion_phys(c, r, &addr, &len);
+        wmb();
+        return addr;
+}
+
+phys_addr_t nvmap_handle_address(struct nvmap_client *c, unsigned long id)
+{
+        struct ion_handle *handle;
+        ion_phys_addr_t addr;
+        size_t len;
+
+        handle = nvmap_convert_handle_u2k(id);
+        ion_phys(c, handle, &addr, &len);
+        return addr;
+}
+
+void nvmap_unpin(struct nvmap_client *client, struct nvmap_handle_ref *r)
+{
+        if (r)
+                ion_handle_put(r);
+}
+
+static int nvmap_reloc_pin_array(struct ion_client *client,
+                                 const struct nvmap_pinarray_elem *arr,
+                                 int nr, struct ion_handle *gather)
+{
+        struct ion_handle *last_patch = NULL;
+        void *patch_addr;
+        ion_phys_addr_t pin_addr;
+        size_t len;
+        int i;
+
+        for (i = 0; i < nr; i++) {
+                struct ion_handle *patch;
+                struct ion_handle *pin;
+                ion_phys_addr_t reloc_addr;
+
+                /* all of the handles are validated and get'ted prior to
+                 * calling this function, so casting is safe here */
+                pin = (struct ion_handle *)arr[i].pin_mem;
+
+                if (arr[i].patch_mem == (unsigned long)last_patch) {
+                        patch = last_patch;
+                } else if (arr[i].patch_mem == (unsigned long)gather) {
+                        patch = gather;
+                } else {
+                        if (last_patch)
+                                ion_handle_put(last_patch);
+
+                        ion_handle_get((struct ion_handle *)arr[i].patch_mem);
+                        patch = (struct ion_handle *)arr[i].patch_mem;
+                        if (!patch)
+                                return -EPERM;
+                        last_patch = patch;
+                }
+
+                patch_addr = ion_map_kernel(client, patch);
+                patch_addr = patch_addr + arr[i].patch_offset;
+
+                ion_phys(client, pin, &pin_addr, &len);
+                reloc_addr = pin_addr + arr[i].pin_offset;
+                __raw_writel(reloc_addr, patch_addr);
+                ion_unmap_kernel(client, patch);
+        }
+
+        if (last_patch)
+                ion_handle_put(last_patch);
+
+        wmb();
+        return 0;
+}
+
+int nvmap_pin_array(struct nvmap_client *client, struct nvmap_handle *gather,
+                    const struct nvmap_pinarray_elem *arr, int nr,
+                    struct nvmap_handle **unique)
+{
+        int i;
+        int count = 0;
+
+        /* FIXME: take care of duplicate ones & validation. */
+        for (i = 0; i < nr; i++) {
+                unique[i] = (struct nvmap_handle *)arr[i].pin_mem;
+                nvmap_pin(client, (struct nvmap_handle_ref *)unique[i]);
+                count++;
+        }
+        nvmap_reloc_pin_array((struct ion_client *)client,
+                arr, nr, (struct ion_handle *)gather);
+        return nr;
+}
+
+void nvmap_unpin_handles(struct nvmap_client *client,
+                         struct nvmap_handle **h, int nr)
+{
+        int i;
+
+        for (i = 0; i < nr; i++)
+                nvmap_unpin(client, h[i]);
+}
+
+int nvmap_patch_word(struct nvmap_client *client,
+                     struct nvmap_handle *patch,
+                     u32 patch_offset, u32 patch_value)
+{
+        void *vaddr;
+        u32 *patch_addr;
+
+        vaddr = ion_map_kernel(client, patch);
+        patch_addr = vaddr + patch_offset;
+        __raw_writel(patch_value, patch_addr);
+        wmb();
+        ion_unmap_kernel(client, patch);
+        return 0;
+}
+
+struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h);
+struct nvmap_handle *nvmap_get_handle_id(struct nvmap_client *client,
+                                         unsigned long id)
+{
+        struct ion_handle *handle;
+
+        handle = (struct ion_handle *)nvmap_convert_handle_u2k(id);
+        pr_debug("id=0x%x, h=0x%x,c=0x%x",
+                (u32)id, (u32)handle, (u32)client);
+        nvmap_handle_get(handle);
+        return handle;
+}
+
+struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
+                                                   unsigned long id)
+{
+        struct ion_buffer *buffer;
+        struct ion_handle *handle;
+        struct ion_client *ion_client = client;
+
+        handle = (struct ion_handle *)nvmap_convert_handle_u2k(id);
+        pr_debug("id=0x%x, h=0x%x,c=0x%x",
+                (u32)id, (u32)handle, (u32)client);
+        buffer = handle->buffer;
+
+        handle = ion_handle_create(client, buffer);
+
+        mutex_lock(&ion_client->lock);
+        ion_handle_add(ion_client, handle);
+        mutex_unlock(&ion_client->lock);
+
+        pr_debug("dup id=0x%x, h=0x%x", (u32)id, (u32)handle);
+        return handle;
+}
+
+void _nvmap_handle_free(struct nvmap_handle *h)
+{
+        ion_handle_put(h);
+}
+
+struct nvmap_handle_ref *nvmap_alloc_iovm(struct nvmap_client *client,
+        size_t size, size_t align, unsigned int flags, unsigned int iova_start)
+{
+        struct ion_handle *h;
+
+        h = ion_alloc(client, size, align, 0xFF);
+        ion_remap_dma(client, h, iova_start);
+        return h;
+}
+
+void nvmap_free_iovm(struct nvmap_client *client, struct nvmap_handle_ref *r)
+{
+        ion_free(client, r);
+}
+
+struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h)
+{
+        ion_handle_get(h);
+        return h;
+}
+
+void nvmap_handle_put(struct nvmap_handle *h)
+{
+        ion_handle_put(h);
+}
+
+#endif
diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c
index 8a1021f2e31..c72f1c0b5e6 100644
--- a/drivers/gpu/vga/vgaarb.c
+++ b/drivers/gpu/vga/vgaarb.c
@@ -1171,10 +1171,9 @@ static int vga_arb_open(struct inode *inode, struct file *file)
 
         pr_debug("%s\n", __func__);
 
-        priv = kmalloc(sizeof(struct vga_arb_private), GFP_KERNEL);
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
         if (priv == NULL)
                 return -ENOMEM;
-        memset(priv, 0, sizeof(*priv));
         spin_lock_init(&priv->lock);
         file->private_data = priv;