87 files changed, 5312 insertions, 1213 deletions
diff --git a/drivers/acpi/video.c b/drivers/acpi/video.c
index db39e9e607d8..ada4b4d9bdc8 100644
--- a/drivers/acpi/video.c
+++ b/drivers/acpi/video.c
@@ -46,7 +46,6 @@
 #define PREFIX "ACPI: "
-#define ACPI_VIDEO_CLASS                "video"
 #define ACPI_VIDEO_BUS_NAME             "Video Bus"
 #define ACPI_VIDEO_DEVICE_NAME          "Video Device"
 #define ACPI_VIDEO_NOTIFY_SWITCH        0x80
@@ -1445,7 +1444,8 @@ static void acpi_video_bus_notify(struct acpi_device *device, u32 event)
        case ACPI_VIDEO_NOTIFY_SWITCH:  /* User requested a switch,
                                         * most likely via hotkey. */
                acpi_bus_generate_proc_event(device, event, 0);
-                keycode = KEY_SWITCHVIDEOMODE;
+                if (!acpi_notifier_call_chain(device, event, 0))
+                        keycode = KEY_SWITCHVIDEOMODE;
                break;
        case ACPI_VIDEO_NOTIFY_PROBE:   /* User plugged in or removed a video
@@ -1475,7 +1475,8 @@ static void acpi_video_bus_notify(struct acpi_device *device, u32 event)
                break;
        }
-        acpi_notifier_call_chain(device, event, 0);
+        if (event != ACPI_VIDEO_NOTIFY_SWITCH)
+                acpi_notifier_call_chain(device, event, 0);
        if (keycode) {
                input_report_key(input, keycode, 1);
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 0a5bea9e3585..987a165ede26 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -1199,6 +1199,26 @@ unsigned int cpufreq_quick_get(unsigned int cpu)
 }
 EXPORT_SYMBOL(cpufreq_quick_get);
+/**
+ * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
+ * @cpu: CPU number
+ *
+ * Just return the max possible frequency for a given CPU.
+ */
+unsigned int cpufreq_quick_get_max(unsigned int cpu)
+{
+        struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+        unsigned int ret_freq = 0;
+        if (policy) {
+                ret_freq = policy->max;
+                cpufreq_cpu_put(policy);
+        }
+        return ret_freq;
+}
+EXPORT_SYMBOL(cpufreq_quick_get_max);
 static unsigned int __cpufreq_get(unsigned int cpu)
 {
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 92369655dca3..f88a9b2c977b 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -560,6 +560,11 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                        mode_changed = true;
                } else if (set->fb == NULL) {
                        mode_changed = true;
+                } else if (set->fb->depth != set->crtc->fb->depth) {
+                        mode_changed = true;
+                } else if (set->fb->bits_per_pixel !=
+                           set->crtc->fb->bits_per_pixel) {
+                        mode_changed = true;
                } else
                        fb_changed = true;
        }
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 4012fe423460..7c5b5f78f1fa 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -211,6 +211,8 @@ drm_gem_handle_delete(struct drm_file *filp, u32 handle)
        idr_remove(&filp->object_idr, handle);
        spin_unlock(&filp->table_lock);
+        if (dev->driver->gem_close_object)
+                dev->driver->gem_close_object(obj, filp);
        drm_gem_object_handle_unreference_unlocked(obj);
        return 0;
@@ -227,7 +229,8 @@ drm_gem_handle_create(struct drm_file *file_priv,
                       struct drm_gem_object *obj,
                       u32 *handlep)
 {
-        int     ret;
+        struct drm_device *dev = obj->dev;
+        int ret;
        /*
         * Get the user-visible handle using idr.
@@ -248,6 +251,15 @@ again:
                return ret;
        drm_gem_object_handle_reference(obj);
+        if (dev->driver->gem_open_object) {
+                ret = dev->driver->gem_open_object(obj, file_priv);
+                if (ret) {
+                        drm_gem_handle_delete(file_priv, *handlep);
+                        return ret;
+                }
+        }
        return 0;
 }
 EXPORT_SYMBOL(drm_gem_handle_create);
@@ -402,7 +414,12 @@ drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
 static int
 drm_gem_object_release_handle(int id, void *ptr, void *data)
 {
+        struct drm_file *file_priv = data;
        struct drm_gem_object *obj = ptr;
+        struct drm_device *dev = obj->dev;
+        if (dev->driver->gem_close_object)
+                dev->driver->gem_close_object(obj, file_priv);
        drm_gem_object_handle_unreference_unlocked(obj);
@@ -418,7 +435,7 @@ void
 drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
 {
        idr_for_each(&file_private->object_idr,
-                     &drm_gem_object_release_handle, NULL);
+                     &drm_gem_object_release_handle, file_private);
        idr_remove_all(&file_private->object_idr);
        idr_destroy(&file_private->object_idr);
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 7223f06d8e58..2a8b6265ad3d 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -123,14 +123,15 @@ static int drm_platform_set_busid(struct drm_device *dev, struct drm_master *mas
 {
        int len, ret;
-        master->unique_len = 10 + strlen(dev->platformdev->name);
+        master->unique_len = 13 + strlen(dev->platformdev->name);
+        master->unique_size = master->unique_len;
        master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
        if (master->unique == NULL)
                return -ENOMEM;
        len = snprintf(master->unique, master->unique_len,
-                       "platform:%s", dev->platformdev->name);
+                        "platform:%s:%02d", dev->platformdev->name, dev->platformdev->id);
        if (len > master->unique_len) {
                DRM_ERROR("Unique buffer overflowed\n");
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 0a893f7400fa..e2662497d50f 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -865,7 +865,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
                           MEMSTAT_VID_SHIFT);
                seq_printf(m, "Current P-state: %d\n",
                           (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
-        } else if (IS_GEN6(dev)) {
+        } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
                u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
                u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
                u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
@@ -1123,6 +1123,44 @@ static int i915_emon_status(struct seq_file *m, void *unused)
        return 0;
 }
+static int i915_ring_freq_table(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int ret;
+        int gpu_freq, ia_freq;
+        if (!(IS_GEN6(dev) || IS_GEN7(dev))) {
+                seq_printf(m, "unsupported on this chipset\n");
+                return 0;
+        }
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+        seq_printf(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\n");
+        for (gpu_freq = dev_priv->min_delay; gpu_freq <= dev_priv->max_delay;
+             gpu_freq++) {
+                I915_WRITE(GEN6_PCODE_DATA, gpu_freq);
+                I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
+                           GEN6_PCODE_READ_MIN_FREQ_TABLE);
+                if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
+                              GEN6_PCODE_READY) == 0, 10)) {
+                        DRM_ERROR("pcode read of freq table timed out\n");
+                        continue;
+                }
+                ia_freq = I915_READ(GEN6_PCODE_DATA);
+                seq_printf(m, "%d\t\t%d\n", gpu_freq * 50, ia_freq * 100);
+        }
+        mutex_unlock(&dev->struct_mutex);
+        return 0;
+}
 static int i915_gfxec(struct seq_file *m, void *unused)
 {
        struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -1430,6 +1468,7 @@ static struct drm_info_list i915_debugfs_list[] = {
        {"i915_inttoext_table", i915_inttoext_table, 0},
        {"i915_drpc_info", i915_drpc_info, 0},
        {"i915_emon_status", i915_emon_status, 0},
+        {"i915_ring_freq_table", i915_ring_freq_table, 0},
        {"i915_gfxec", i915_gfxec, 0},
        {"i915_fbc_status", i915_fbc_status, 0},
        {"i915_sr_status", i915_sr_status, 0},
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 7eef6e11d9ac..8a3942c4f099 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -1071,6 +1071,9 @@ static void i915_setup_compression(struct drm_device *dev, int size)
        unsigned long cfb_base;
        unsigned long ll_base = 0;
+        /* Just in case the BIOS is doing something questionable. */
+        intel_disable_fbc(dev);
        compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
        if (compressed_fb)
                compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
@@ -1097,7 +1100,6 @@ static void i915_setup_compression(struct drm_device *dev, int size)
        dev_priv->cfb_size = size;
-        intel_disable_fbc(dev);
        dev_priv->compressed_fb = compressed_fb;
        if (HAS_PCH_SPLIT(dev))
                I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index eb91e2dd7914..ce045a8cf82c 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -37,38 +37,70 @@
 #include <linux/console.h>
 #include "drm_crtc_helper.h"
-static int i915_modeset = -1;
+static int i915_modeset __read_mostly = -1;
 module_param_named(modeset, i915_modeset, int, 0400);
+MODULE_PARM_DESC(modeset,
+                "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, "
+                "1=on, -1=force vga console preference [default])");
-unsigned int i915_fbpercrtc = 0;
+unsigned int i915_fbpercrtc __always_unused = 0;
 module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400);
-int i915_panel_ignore_lid = 0;
+int i915_panel_ignore_lid __read_mostly = 0;
 module_param_named(panel_ignore_lid, i915_panel_ignore_lid, int, 0600);
+MODULE_PARM_DESC(panel_ignore_lid,
+                "Override lid status (0=autodetect [default], 1=lid open, "
+                "-1=lid closed)");
-unsigned int i915_powersave = 1;
+unsigned int i915_powersave __read_mostly = 1;
 module_param_named(powersave, i915_powersave, int, 0600);
+MODULE_PARM_DESC(powersave,
+                "Enable powersavings, fbc, downclocking, etc. (default: true)");
-unsigned int i915_semaphores = 0;
+unsigned int i915_semaphores __read_mostly = 0;
 module_param_named(semaphores, i915_semaphores, int, 0600);
+MODULE_PARM_DESC(semaphores,
+                "Use semaphores for inter-ring sync (default: false)");
-unsigned int i915_enable_rc6 = 0;
+unsigned int i915_enable_rc6 __read_mostly = 0;
 module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0600);
+MODULE_PARM_DESC(i915_enable_rc6,
+                "Enable power-saving render C-state 6 (default: true)");
-unsigned int i915_enable_fbc = 0;
+unsigned int i915_enable_fbc __read_mostly = 1;
 module_param_named(i915_enable_fbc, i915_enable_fbc, int, 0600);
+MODULE_PARM_DESC(i915_enable_fbc,
+                "Enable frame buffer compression for power savings "
+                "(default: false)");
-unsigned int i915_lvds_downclock = 0;
+unsigned int i915_lvds_downclock __read_mostly = 0;
 module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400);
+MODULE_PARM_DESC(lvds_downclock,
+                "Use panel (LVDS/eDP) downclocking for power savings "
+                "(default: false)");
-unsigned int i915_panel_use_ssc = 1;
+unsigned int i915_panel_use_ssc __read_mostly = 1;
 module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600);
+MODULE_PARM_DESC(lvds_use_ssc,
+                "Use Spread Spectrum Clock with panels [LVDS/eDP] "
+                "(default: true)");
-int i915_vbt_sdvo_panel_type = -1;
+int i915_vbt_sdvo_panel_type __read_mostly = -1;
 module_param_named(vbt_sdvo_panel_type, i915_vbt_sdvo_panel_type, int, 0600);
+MODULE_PARM_DESC(vbt_sdvo_panel_type,
+                "Override selection of SDVO panel mode in the VBT "
+                "(default: auto)");
-static bool i915_try_reset = true;
+static bool i915_try_reset __read_mostly = true;
 module_param_named(reset, i915_try_reset, bool, 0600);
+MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
+bool i915_enable_hangcheck __read_mostly = true;
+module_param_named(enable_hangcheck, i915_enable_hangcheck, bool, 0644);
+MODULE_PARM_DESC(enable_hangcheck,
+                "Periodically check GPU activity for detecting hangs. "
+                "WARNING: Disabling this can cause system wide hangs. "
+                "(default: true)");
 static struct drm_driver driver;
 extern int intel_agp_enabled;
@@ -345,12 +377,17 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 {
-        int loop = 500;
+        if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES ) {
-        u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+                int loop = 500;
-        while (fifo < 20 && loop--) {
+                u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
-                udelay(10);
+                while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
-                fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+                        udelay(10);
+                        fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
+                }
+                WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES);
+                dev_priv->gt_fifo_count = fifo;
        }
+        dev_priv->gt_fifo_count--;
 }
 static int i915_drm_freeze(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index ce7914c4c044..6867e193d85e 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -214,6 +214,8 @@ struct drm_i915_display_funcs {
        int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
                          struct drm_framebuffer *fb,
                          struct drm_i915_gem_object *obj);
+        int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                            int x, int y);
        /* clock updates for mode set */
        /* cursor updates */
        /* render clock increase/decrease */
@@ -265,6 +267,7 @@ enum intel_pch {
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
 struct intel_fbdev;
+struct intel_fbc_work;
 typedef struct drm_i915_private {
        struct drm_device *dev;
@@ -275,6 +278,7 @@ typedef struct drm_i915_private {
        int relative_constants_mode;
        void __iomem *regs;
+        u32 gt_fifo_count;
        struct intel_gmbus {
                struct i2c_adapter adapter;
@@ -329,11 +333,10 @@ typedef struct drm_i915_private {
        uint32_t last_instdone1;
        unsigned long cfb_size;
-        unsigned long cfb_pitch;
+        unsigned int cfb_fb;
-        unsigned long cfb_offset;
+        enum plane cfb_plane;
-        int cfb_fence;
-        int cfb_plane;
        int cfb_y;
+        struct intel_fbc_work *fbc_work;
        struct intel_opregion opregion;
@@ -986,15 +989,16 @@ struct drm_i915_file_private {
 extern struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
-extern unsigned int i915_fbpercrtc;
+extern unsigned int i915_fbpercrtc __always_unused;
-extern int i915_panel_ignore_lid;
+extern int i915_panel_ignore_lid __read_mostly;
-extern unsigned int i915_powersave;
+extern unsigned int i915_powersave __read_mostly;
-extern unsigned int i915_semaphores;
+extern unsigned int i915_semaphores __read_mostly;
-extern unsigned int i915_lvds_downclock;
+extern unsigned int i915_lvds_downclock __read_mostly;
-extern unsigned int i915_panel_use_ssc;
+extern unsigned int i915_panel_use_ssc __read_mostly;
-extern int i915_vbt_sdvo_panel_type;
+extern int i915_vbt_sdvo_panel_type __read_mostly;
-extern unsigned int i915_enable_rc6;
+extern unsigned int i915_enable_rc6 __read_mostly;
-extern unsigned int i915_enable_fbc;
+extern unsigned int i915_enable_fbc __read_mostly;
+extern bool i915_enable_hangcheck __read_mostly;
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
@@ -1164,7 +1168,7 @@ void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
                                            uint32_t read_domains,
                                            uint32_t write_domain);
-int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
 void i915_gem_do_init(struct drm_device *dev,
@@ -1183,7 +1187,8 @@ int __must_check
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
                                  bool write);
 int __must_check
-i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
+                                     u32 alignment,
                                     struct intel_ring_buffer *pipelined);
 int i915_gem_attach_phys_object(struct drm_device *dev,
                                struct drm_i915_gem_object *obj,
@@ -1199,9 +1204,14 @@ i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
                                    uint32_t size,
                                    int tiling_mode);
+int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
+                                    enum i915_cache_level cache_level);
 /* i915_gem_gtt.c */
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
+void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
+                                enum i915_cache_level cache_level);
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 /* i915_gem_evict.c */
@@ -1283,12 +1293,8 @@ extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
-extern void i8xx_disable_fbc(struct drm_device *dev);
-extern void g4x_disable_fbc(struct drm_device *dev);
-extern void ironlake_disable_fbc(struct drm_device *dev);
-extern void intel_disable_fbc(struct drm_device *dev);
-extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval);
 extern bool intel_fbc_enabled(struct drm_device *dev);
+extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
 extern void ironlake_enable_rc6(struct drm_device *dev);
 extern void gen6_set_rps(struct drm_device *dev, u8 val);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index a087e1bf0c2f..a546a71fb060 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1763,8 +1763,11 @@ i915_add_request(struct intel_ring_buffer *ring,
        ring->outstanding_lazy_request = false;
        if (!dev_priv->mm.suspended) {
-                mod_timer(&dev_priv->hangcheck_timer,
+                if (i915_enable_hangcheck) {
-                          jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+                        mod_timer(&dev_priv->hangcheck_timer,
+                                  jiffies +
+                                  msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+                }
                if (was_empty)
                        queue_delayed_work(dev_priv->wq,
                                           &dev_priv->mm.retire_work, HZ);
@@ -2135,6 +2138,30 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
        return 0;
 }
+static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
+{
+        u32 old_write_domain, old_read_domains;
+        /* Act a barrier for all accesses through the GTT */
+        mb();
+        /* Force a pagefault for domain tracking on next user access */
+        i915_gem_release_mmap(obj);
+        if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
+                return;
+        old_read_domains = obj->base.read_domains;
+        old_write_domain = obj->base.write_domain;
+        obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
+        obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
+        trace_i915_gem_object_change_domain(obj,
+                                            old_read_domains,
+                                            old_write_domain);
+}
 /**
 * Unbinds an object from the GTT aperture.
 */
@@ -2151,23 +2178,28 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
                return -EINVAL;
        }
-        /* blow away mappings if mapped through GTT */
+        ret = i915_gem_object_finish_gpu(obj);
-        i915_gem_release_mmap(obj);
-        /* Move the object to the CPU domain to ensure that
-         * any possible CPU writes while it's not in the GTT
-         * are flushed when we go to remap it. This will
-         * also ensure that all pending GPU writes are finished
-         * before we unbind.
-         */
-        ret = i915_gem_object_set_to_cpu_domain(obj, 1);
        if (ret == -ERESTARTSYS)
                return ret;
        /* Continue on if we fail due to EIO, the GPU is hung so we
         * should be safe and we need to cleanup or else we might
         * cause memory corruption through use-after-free.
         */
+        i915_gem_object_finish_gtt(obj);
+        /* Move the object to the CPU domain to ensure that
+         * any possible CPU writes while it's not in the GTT
+         * are flushed when we go to remap it.
+         */
+        if (ret == 0)
+                ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+        if (ret == -ERESTARTSYS)
+                return ret;
        if (ret) {
+                /* In the event of a disaster, abandon all caches and
+                 * hope for the best.
+                 */
                i915_gem_clflush_object(obj);
                obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
        }
@@ -2996,51 +3028,139 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
        return 0;
 }
+int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
+                                    enum i915_cache_level cache_level)
+{
+        int ret;
+        if (obj->cache_level == cache_level)
+                return 0;
+        if (obj->pin_count) {
+                DRM_DEBUG("can not change the cache level of pinned objects\n");
+                return -EBUSY;
+        }
+        if (obj->gtt_space) {
+                ret = i915_gem_object_finish_gpu(obj);
+                if (ret)
+                        return ret;
+                i915_gem_object_finish_gtt(obj);
+                /* Before SandyBridge, you could not use tiling or fence
+                 * registers with snooped memory, so relinquish any fences
+                 * currently pointing to our region in the aperture.
+                 */
+                if (INTEL_INFO(obj->base.dev)->gen < 6) {
+                        ret = i915_gem_object_put_fence(obj);
+                        if (ret)
+                                return ret;
+                }
+                i915_gem_gtt_rebind_object(obj, cache_level);
+        }
+        if (cache_level == I915_CACHE_NONE) {
+                u32 old_read_domains, old_write_domain;
+                /* If we're coming from LLC cached, then we haven't
+                 * actually been tracking whether the data is in the
+                 * CPU cache or not, since we only allow one bit set
+                 * in obj->write_domain and have been skipping the clflushes.
+                 * Just set it to the CPU cache for now.
+                 */
+                WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
+                WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
+                old_read_domains = obj->base.read_domains;
+                old_write_domain = obj->base.write_domain;
+                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+                trace_i915_gem_object_change_domain(obj,
+                                                    old_read_domains,
+                                                    old_write_domain);
+        }
+        obj->cache_level = cache_level;
+        return 0;
+}
 /*
- * Prepare buffer for display plane. Use uninterruptible for possible flush
+ * Prepare buffer for display plane (scanout, cursors, etc).
- * wait, as in modesetting process we're not supposed to be interrupted.
+ * Can be called from an uninterruptible phase (modesetting) and allows
+ * any flushes to be pipelined (for pageflips).
+ *
+ * For the display plane, we want to be in the GTT but out of any write
+ * domains. So in many ways this looks like set_to_gtt_domain() apart from the
+ * ability to pipeline the waits, pinning and any additional subtleties
+ * that may differentiate the display plane from ordinary buffers.
 */
 int
-i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
+i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
+                                     u32 alignment,
                                     struct intel_ring_buffer *pipelined)
 {
-        uint32_t old_read_domains;
+        u32 old_read_domains, old_write_domain;
        int ret;
-        /* Not valid to be called on unbound objects. */
-        if (obj->gtt_space == NULL)
-                return -EINVAL;
        ret = i915_gem_object_flush_gpu_write_domain(obj);
        if (ret)
                return ret;
-        /* Currently, we are always called from an non-interruptible context. */
        if (pipelined != obj->ring) {
                ret = i915_gem_object_wait_rendering(obj);
-                if (ret)
+                if (ret == -ERESTARTSYS)
                        return ret;
        }
+        /* The display engine is not coherent with the LLC cache on gen6.  As
+         * a result, we make sure that the pinning that is about to occur is
+         * done with uncached PTEs. This is lowest common denominator for all
+         * chipsets.
+         *
+         * However for gen6+, we could do better by using the GFDT bit instead
+         * of uncaching, which would allow us to flush all the LLC-cached data
+         * with that bit in the PTE to main memory with just one PIPE_CONTROL.
+         */
+        ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
+        if (ret)
+                return ret;
+        /* As the user may map the buffer once pinned in the display plane
+         * (e.g. libkms for the bootup splash), we have to ensure that we
+         * always use map_and_fenceable for all scanout buffers.
+         */
+        ret = i915_gem_object_pin(obj, alignment, true);
+        if (ret)
+                return ret;
        i915_gem_object_flush_cpu_write_domain(obj);
+        old_write_domain = obj->base.write_domain;
        old_read_domains = obj->base.read_domains;
+        /* It should now be out of any other write domains, and we can update
+         * the domain values for our changes.
+         */
+        BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
        obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
        trace_i915_gem_object_change_domain(obj,
                                            old_read_domains,
-                                            obj->base.write_domain);
+                                            old_write_domain);
        return 0;
 }
 int
-i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
+i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
 {
        int ret;
-        if (!obj->active)
+        if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
                return 0;
        if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
@@ -3049,6 +3169,9 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
                        return ret;
        }
+        /* Ensure that we invalidate the GPU's caches and TLBs. */
+        obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
        return i915_gem_object_wait_rendering(obj);
 }
@@ -3575,7 +3698,23 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
        obj->base.write_domain = I915_GEM_DOMAIN_CPU;
        obj->base.read_domains = I915_GEM_DOMAIN_CPU;
-        obj->cache_level = I915_CACHE_NONE;
+        if (IS_GEN6(dev)) {
+                /* On Gen6, we can have the GPU use the LLC (the CPU
+                 * cache) for about a 10% performance improvement
+                 * compared to uncached.  Graphics requests other than
+                 * display scanout are coherent with the CPU in
+                 * accessing this cache.  This means in this mode we
+                 * don't need to clflush on the CPU side, and on the
+                 * GPU side we only need to flush internal caches to
+                 * get data visible to the CPU.
+                 *
+                 * However, we maintain the display planes as UC, and so
+                 * need to rebind when first used as such.
+                 */
+                obj->cache_level = I915_CACHE_LLC;
+        } else
+                obj->cache_level = I915_CACHE_NONE;
        obj->base.driver_private = NULL;
        obj->fence_reg = I915_FENCE_REG_NONE;
        INIT_LIST_HEAD(&obj->mm_list);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index e46b645773cf..7a709cd8d543 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -59,24 +59,8 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
                              (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
-                unsigned int agp_type =
-                        cache_level_to_agp_type(dev, obj->cache_level);
                i915_gem_clflush_object(obj);
+                i915_gem_gtt_rebind_object(obj, obj->cache_level);
-                if (dev_priv->mm.gtt->needs_dmar) {
-                        BUG_ON(!obj->sg_list);
-                        intel_gtt_insert_sg_entries(obj->sg_list,
-                                                    obj->num_sg,
-                                                    obj->gtt_space->start >> PAGE_SHIFT,
-                                                    agp_type);
-                } else
-                        intel_gtt_insert_pages(obj->gtt_space->start
-                                                   >> PAGE_SHIFT,
-                                               obj->base.size >> PAGE_SHIFT,
-                                               obj->pages,
-                                               agp_type);
        }
        intel_gtt_chipset_flush();
@@ -110,6 +94,27 @@ int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
        return 0;
 }
+void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
+                                enum i915_cache_level cache_level)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
+        if (dev_priv->mm.gtt->needs_dmar) {
+                BUG_ON(!obj->sg_list);
+                intel_gtt_insert_sg_entries(obj->sg_list,
+                                            obj->num_sg,
+                                            obj->gtt_space->start >> PAGE_SHIFT,
+                                            agp_type);
+        } else
+                intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
+                                       obj->base.size >> PAGE_SHIFT,
+                                       obj->pages,
+                                       agp_type);
+}
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 {
        intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 9da2a2c99953..7ef3e8b6864d 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -364,10 +364,12 @@ static void notify_ring(struct drm_device *dev,
        ring->irq_seqno = seqno;
        wake_up_all(&ring->irq_queue);
+        if (i915_enable_hangcheck) {
-        dev_priv->hangcheck_count = 0;
+                dev_priv->hangcheck_count = 0;
-        mod_timer(&dev_priv->hangcheck_timer,
+                mod_timer(&dev_priv->hangcheck_timer,
-                  jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+                          jiffies +
+                          msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
+        }
 }
 static void gen6_pm_rps_work(struct work_struct *work)
@@ -1667,6 +1669,9 @@ void i915_hangcheck_elapsed(unsigned long data)
        uint32_t acthd, instdone, instdone1;
        bool err = false;
+        if (!i915_enable_hangcheck)
+                return;
        /* If all work is done then ACTHD clearly hasn't advanced. */
        if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
            i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 5d5def756c9e..02db299f621a 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -579,6 +579,7 @@
 #define   DPFC_CTL_PLANEA       (0<<30)
 #define   DPFC_CTL_PLANEB       (1<<30)
 #define   DPFC_CTL_FENCE_EN     (1<<29)
+#define   DPFC_CTL_PERSISTENT_MODE      (1<<25)
 #define   DPFC_SR_EN            (1<<10)
 #define   DPFC_CTL_LIMIT_1X     (0<<6)
 #define   DPFC_CTL_LIMIT_2X     (1<<6)
@@ -3360,6 +3361,7 @@
 #define  FORCEWAKE_ACK                          0x130090
 #define  GT_FIFO_FREE_ENTRIES                   0x120008
+#define    GT_FIFO_NUM_RESERVED_ENTRIES         20
 #define GEN6_RPNSWREQ                           0xA008
 #define   GEN6_TURBO_DISABLE                    (1<<31)
@@ -3434,7 +3436,9 @@
 #define GEN6_PCODE_MAILBOX                      0x138124
 #define   GEN6_PCODE_READY                      (1<<31)
 #define   GEN6_READ_OC_PARAMS                   0xc
-#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE       0x9
+#define   GEN6_PCODE_WRITE_MIN_FREQ_TABLE       0x8
+#define   GEN6_PCODE_READ_MIN_FREQ_TABLE        0x9
 #define GEN6_PCODE_DATA                         0x138128
+#define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT        8
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 5257cfc34c35..285758603ac8 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -760,15 +760,13 @@ static void i915_restore_display(struct drm_device *dev)
        /* FIXME: restore TV & SDVO state */
        /* only restore FBC info on the platform that supports FBC*/
+        intel_disable_fbc(dev);
        if (I915_HAS_FBC(dev)) {
                if (HAS_PCH_SPLIT(dev)) {
-                        ironlake_disable_fbc(dev);
                        I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->saveDPFC_CB_BASE);
                } else if (IS_GM45(dev)) {
-                        g4x_disable_fbc(dev);
                        I915_WRITE(DPFC_CB_BASE, dev_priv->saveDPFC_CB_BASE);
                } else {
-                        i8xx_disable_fbc(dev);
                        I915_WRITE(FBC_CFB_BASE, dev_priv->saveFBC_CFB_BASE);
                        I915_WRITE(FBC_LL_BASE, dev_priv->saveFBC_LL_BASE);
                        I915_WRITE(FBC_CONTROL2, dev_priv->saveFBC_CONTROL2);
@@ -878,8 +876,10 @@ int i915_restore_state(struct drm_device *dev)
                intel_init_emon(dev);
        }
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev)) {
                gen6_enable_rps(dev_priv);
+                gen6_update_ring_freq(dev_priv);
+        }
        mutex_lock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 927442a11925..61abef8a8119 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -74,7 +74,7 @@ get_blocksize(void *p)
 static void
 fill_detail_timing_data(struct drm_display_mode *panel_fixed_mode,
-                        struct lvds_dvo_timing *dvo_timing)
+                        const struct lvds_dvo_timing *dvo_timing)
 {
        panel_fixed_mode->hdisplay = (dvo_timing->hactive_hi << 8) |
                dvo_timing->hactive_lo;
@@ -115,20 +115,75 @@ fill_detail_timing_data(struct drm_display_mode *panel_fixed_mode,
        drm_mode_set_name(panel_fixed_mode);
 }
+static bool
+lvds_dvo_timing_equal_size(const struct lvds_dvo_timing *a,
+                           const struct lvds_dvo_timing *b)
+{
+        if (a->hactive_hi != b->hactive_hi ||
+            a->hactive_lo != b->hactive_lo)
+                return false;
+        if (a->hsync_off_hi != b->hsync_off_hi ||
+            a->hsync_off_lo != b->hsync_off_lo)
+                return false;
+        if (a->hsync_pulse_width != b->hsync_pulse_width)
+                return false;
+        if (a->hblank_hi != b->hblank_hi ||
+            a->hblank_lo != b->hblank_lo)
+                return false;
+        if (a->vactive_hi != b->vactive_hi ||
+            a->vactive_lo != b->vactive_lo)
+                return false;
+        if (a->vsync_off != b->vsync_off)
+                return false;
+        if (a->vsync_pulse_width != b->vsync_pulse_width)
+                return false;
+        if (a->vblank_hi != b->vblank_hi ||
+            a->vblank_lo != b->vblank_lo)
+                return false;
+        return true;
+}
+static const struct lvds_dvo_timing *
+get_lvds_dvo_timing(const struct bdb_lvds_lfp_data *lvds_lfp_data,
+                    const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs,
+                    int index)
+{
+        /*
+         * the size of fp_timing varies on the different platform.
+         * So calculate the DVO timing relative offset in LVDS data
+         * entry to get the DVO timing entry
+         */
+        int lfp_data_size =
+                lvds_lfp_data_ptrs->ptr[1].dvo_timing_offset -
+                lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset;
+        int dvo_timing_offset =
+                lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset -
+                lvds_lfp_data_ptrs->ptr[0].fp_timing_offset;
+        char *entry = (char *)lvds_lfp_data->data + lfp_data_size * index;
+        return (struct lvds_dvo_timing *)(entry + dvo_timing_offset);
+}
 /* Try to find integrated panel data */
 static void
 parse_lfp_panel_data(struct drm_i915_private *dev_priv,
                            struct bdb_header *bdb)
 {
-        struct bdb_lvds_options *lvds_options;
+        const struct bdb_lvds_options *lvds_options;
-        struct bdb_lvds_lfp_data *lvds_lfp_data;
+        const struct bdb_lvds_lfp_data *lvds_lfp_data;
-        struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs;
+        const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs;
-        struct bdb_lvds_lfp_data_entry *entry;
+        const struct lvds_dvo_timing *panel_dvo_timing;
-        struct lvds_dvo_timing *dvo_timing;
        struct drm_display_mode *panel_fixed_mode;
-        int lfp_data_size, dvo_timing_offset;
+        int i, downclock;
-        int i, temp_downclock;
-        struct drm_display_mode *temp_mode;
        lvds_options = find_section(bdb, BDB_LVDS_OPTIONS);
        if (!lvds_options)
@@ -150,75 +205,44 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
        dev_priv->lvds_vbt = 1;
-        lfp_data_size = lvds_lfp_data_ptrs->ptr[1].dvo_timing_offset -
+        panel_dvo_timing = get_lvds_dvo_timing(lvds_lfp_data,
-                lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset;
+                                               lvds_lfp_data_ptrs,
-        entry = (struct bdb_lvds_lfp_data_entry *)
+                                               lvds_options->panel_type);
-                ((uint8_t *)lvds_lfp_data->data + (lfp_data_size *
-                                                   lvds_options->panel_type));
-        dvo_timing_offset = lvds_lfp_data_ptrs->ptr[0].dvo_timing_offset -
-                lvds_lfp_data_ptrs->ptr[0].fp_timing_offset;
-        /*
-         * the size of fp_timing varies on the different platform.
-         * So calculate the DVO timing relative offset in LVDS data
-         * entry to get the DVO timing entry
-         */
-        dvo_timing = (struct lvds_dvo_timing *)
-                        ((unsigned char *)entry + dvo_timing_offset);
        panel_fixed_mode = kzalloc(sizeof(*panel_fixed_mode), GFP_KERNEL);
        if (!panel_fixed_mode)
                return;
-        fill_detail_timing_data(panel_fixed_mode, dvo_timing);
+        fill_detail_timing_data(panel_fixed_mode, panel_dvo_timing);
        dev_priv->lfp_lvds_vbt_mode = panel_fixed_mode;
        DRM_DEBUG_KMS("Found panel mode in BIOS VBT tables:\n");
        drm_mode_debug_printmodeline(panel_fixed_mode);
-        temp_mode = kzalloc(sizeof(*temp_mode), GFP_KERNEL);
-        temp_downclock = panel_fixed_mode->clock;
        /*
-         * enumerate the LVDS panel timing info entry in VBT to check whether
+         * Iterate over the LVDS panel timing info to find the lowest clock
-         * the LVDS downclock is found.
+         * for the native resolution.
         */
+        downclock = panel_dvo_timing->clock;
        for (i = 0; i < 16; i++) {
-                entry = (struct bdb_lvds_lfp_data_entry *)
+                const struct lvds_dvo_timing *dvo_timing;
-                        ((uint8_t *)lvds_lfp_data->data + (lfp_data_size * i));
-                dvo_timing = (struct lvds_dvo_timing *)
+                dvo_timing = get_lvds_dvo_timing(lvds_lfp_data,
-                        ((unsigned char *)entry + dvo_timing_offset);
+                                                 lvds_lfp_data_ptrs,
+                                                 i);
-                fill_detail_timing_data(temp_mode, dvo_timing);
+                if (lvds_dvo_timing_equal_size(dvo_timing, panel_dvo_timing) &&
+                    dvo_timing->clock < downclock)
-                if (temp_mode->hdisplay == panel_fixed_mode->hdisplay &&
+                        downclock = dvo_timing->clock;
-                temp_mode->hsync_start == panel_fixed_mode->hsync_start &&
-                temp_mode->hsync_end == panel_fixed_mode->hsync_end &&
-                temp_mode->htotal == panel_fixed_mode->htotal &&
-                temp_mode->vdisplay == panel_fixed_mode->vdisplay &&
-                temp_mode->vsync_start == panel_fixed_mode->vsync_start &&
-                temp_mode->vsync_end == panel_fixed_mode->vsync_end &&
-                temp_mode->vtotal == panel_fixed_mode->vtotal &&
-                temp_mode->clock < temp_downclock) {
-                        /*
-                         * downclock is already found. But we expect
-                         * to find the lower downclock.
-                         */
-                        temp_downclock = temp_mode->clock;
-                }
-                /* clear it to zero */
-                memset(temp_mode, 0, sizeof(*temp_mode));
        }
-        kfree(temp_mode);
-        if (temp_downclock < panel_fixed_mode->clock &&
+        if (downclock < panel_dvo_timing->clock && i915_lvds_downclock) {
-            i915_lvds_downclock) {
                dev_priv->lvds_downclock_avail = 1;
-                dev_priv->lvds_downclock = temp_downclock;
+                dev_priv->lvds_downclock = downclock * 10;
                DRM_DEBUG_KMS("LVDS downclock is found in VBT. "
                              "Normal Clock %dKHz, downclock %dKHz\n",
-                              temp_downclock, panel_fixed_mode->clock);
+                              panel_fixed_mode->clock, 10*downclock);
        }
-        return;
 }
 /* Try to find sdvo panel data */
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 5609c065aaf4..97d28013db79 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -24,6 +24,7 @@
 *      Eric Anholt <eric@anholt.net>
 */
+#include <linux/cpufreq.h>
 #include <linux/module.h>
 #include <linux/input.h>
 #include <linux/i2c.h>
@@ -1157,12 +1158,15 @@ static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
        reg = TRANSCONF(pipe);
        val = I915_READ(reg);
-        /*
-         * make the BPC in transcoder be consistent with
+        if (HAS_PCH_IBX(dev_priv->dev)) {
-         * that in pipeconf reg.
+                /*
-         */
+                 * make the BPC in transcoder be consistent with
-        val &= ~PIPE_BPC_MASK;
+                 * that in pipeconf reg.
-        val |= I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK;
+                 */
+                val &= ~PIPE_BPC_MASK;
+                val |= I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK;
+        }
        I915_WRITE(reg, val | TRANS_ENABLE);
        if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
                DRM_ERROR("failed to enable transcoder %d\n", pipe);
@@ -1380,6 +1384,28 @@ static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
        disable_pch_hdmi(dev_priv, pipe, HDMID);
 }
+static void i8xx_disable_fbc(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 fbc_ctl;
+        /* Disable compression */
+        fbc_ctl = I915_READ(FBC_CONTROL);
+        if ((fbc_ctl & FBC_CTL_EN) == 0)
+                return;
+        fbc_ctl &= ~FBC_CTL_EN;
+        I915_WRITE(FBC_CONTROL, fbc_ctl);
+        /* Wait for compressing bit to clear */
+        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
+                DRM_DEBUG_KMS("FBC idle timed out\n");
+                return;
+        }
+        DRM_DEBUG_KMS("disabled FBC\n");
+}
 static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 {
        struct drm_device *dev = crtc->dev;
@@ -1388,36 +1414,25 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
        struct drm_i915_gem_object *obj = intel_fb->obj;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int cfb_pitch;
        int plane, i;
        u32 fbc_ctl, fbc_ctl2;
-        if (fb->pitch == dev_priv->cfb_pitch &&
+        cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
-            obj->fence_reg == dev_priv->cfb_fence &&
+        if (fb->pitch < cfb_pitch)
-            intel_crtc->plane == dev_priv->cfb_plane &&
+                cfb_pitch = fb->pitch;
-            I915_READ(FBC_CONTROL) & FBC_CTL_EN)
-                return;
-        i8xx_disable_fbc(dev);
-        dev_priv->cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
-        if (fb->pitch < dev_priv->cfb_pitch)
-                dev_priv->cfb_pitch = fb->pitch;
        /* FBC_CTL wants 64B units */
-        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
+        cfb_pitch = (cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj->fence_reg;
+        plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
-        dev_priv->cfb_plane = intel_crtc->plane;
-        plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
        /* Clear old tags */
        for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
                I915_WRITE(FBC_TAG + (i * 4), 0);
        /* Set it up... */
-        fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
+        fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
-        if (obj->tiling_mode != I915_TILING_NONE)
+        fbc_ctl2 |= plane;
-                fbc_ctl2 |= FBC_CTL_CPU_FENCE;
        I915_WRITE(FBC_CONTROL2, fbc_ctl2);
        I915_WRITE(FBC_FENCE_OFF, crtc->y);
@@ -1425,36 +1440,13 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
        if (IS_I945GM(dev))
                fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
-        fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
+        fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
        fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
-        if (obj->tiling_mode != I915_TILING_NONE)
+        fbc_ctl |= obj->fence_reg;
-                fbc_ctl |= dev_priv->cfb_fence;
-        I915_WRITE(FBC_CONTROL, fbc_ctl);
-        DRM_DEBUG_KMS("enabled FBC, pitch %ld, yoff %d, plane %d, ",
-                      dev_priv->cfb_pitch, crtc->y, dev_priv->cfb_plane);
-}
-void i8xx_disable_fbc(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 fbc_ctl;
-        /* Disable compression */
-        fbc_ctl = I915_READ(FBC_CONTROL);
-        if ((fbc_ctl & FBC_CTL_EN) == 0)
-                return;
-        fbc_ctl &= ~FBC_CTL_EN;
        I915_WRITE(FBC_CONTROL, fbc_ctl);
-        /* Wait for compressing bit to clear */
+        DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
-        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
+                      cfb_pitch, crtc->y, intel_crtc->plane);
-                DRM_DEBUG_KMS("FBC idle timed out\n");
-                return;
-        }
-        DRM_DEBUG_KMS("disabled FBC\n");
 }
 static bool i8xx_fbc_enabled(struct drm_device *dev)
@@ -1476,30 +1468,9 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        unsigned long stall_watermark = 200;
        u32 dpfc_ctl;
-        dpfc_ctl = I915_READ(DPFC_CONTROL);
-        if (dpfc_ctl & DPFC_CTL_EN) {
-                if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
-                    dev_priv->cfb_fence == obj->fence_reg &&
-                    dev_priv->cfb_plane == intel_crtc->plane &&
-                    dev_priv->cfb_y == crtc->y)
-                        return;
-                I915_WRITE(DPFC_CONTROL, dpfc_ctl & ~DPFC_CTL_EN);
-                intel_wait_for_vblank(dev, intel_crtc->pipe);
-        }
-        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj->fence_reg;
-        dev_priv->cfb_plane = intel_crtc->plane;
-        dev_priv->cfb_y = crtc->y;
        dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
-        if (obj->tiling_mode != I915_TILING_NONE) {
+        dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
-                dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
+        I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
-                I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
-        } else {
-                I915_WRITE(DPFC_CHICKEN, ~DPFC_HT_MODIFY);
-        }
        I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
                   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
@@ -1512,7 +1483,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
 }
-void g4x_disable_fbc(struct drm_device *dev)
+static void g4x_disable_fbc(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        u32 dpfc_ctl;
@@ -1567,32 +1538,12 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        u32 dpfc_ctl;
        dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
-        if (dpfc_ctl & DPFC_CTL_EN) {
-                if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
-                    dev_priv->cfb_fence == obj->fence_reg &&
-                    dev_priv->cfb_plane == intel_crtc->plane &&
-                    dev_priv->cfb_offset == obj->gtt_offset &&
-                    dev_priv->cfb_y == crtc->y)
-                        return;
-                I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl & ~DPFC_CTL_EN);
-                intel_wait_for_vblank(dev, intel_crtc->pipe);
-        }
-        dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
-        dev_priv->cfb_fence = obj->fence_reg;
-        dev_priv->cfb_plane = intel_crtc->plane;
-        dev_priv->cfb_offset = obj->gtt_offset;
-        dev_priv->cfb_y = crtc->y;
        dpfc_ctl &= DPFC_RESERVED;
        dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
-        if (obj->tiling_mode != I915_TILING_NONE) {
+        /* Set persistent mode for front-buffer rendering, ala X. */
-                dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence);
+        dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
-                I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
+        dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
-        } else {
+        I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
-                I915_WRITE(ILK_DPFC_CHICKEN, ~DPFC_HT_MODIFY);
-        }
        I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
                   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
@@ -1604,7 +1555,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        if (IS_GEN6(dev)) {
                I915_WRITE(SNB_DPFC_CTL_SA,
-                           SNB_CPU_FENCE_ENABLE | dev_priv->cfb_fence);
+                           SNB_CPU_FENCE_ENABLE | obj->fence_reg);
                I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
                sandybridge_blit_fbc_update(dev);
        }
@@ -1612,7 +1563,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
        DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
 }
-void ironlake_disable_fbc(struct drm_device *dev)
+static void ironlake_disable_fbc(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
        u32 dpfc_ctl;
@@ -1644,24 +1595,109 @@ bool intel_fbc_enabled(struct drm_device *dev)
        return dev_priv->display.fbc_enabled(dev);
 }
-void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+static void intel_fbc_work_fn(struct work_struct *__work)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct intel_fbc_work *work =
+                container_of(to_delayed_work(__work),
+                             struct intel_fbc_work, work);
+        struct drm_device *dev = work->crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        mutex_lock(&dev->struct_mutex);
+        if (work == dev_priv->fbc_work) {
+                /* Double check that we haven't switched fb without cancelling
+                 * the prior work.
+                 */
+                if (work->crtc->fb == work->fb) {
+                        dev_priv->display.enable_fbc(work->crtc,
+                                                     work->interval);
+                        dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
+                        dev_priv->cfb_fb = work->crtc->fb->base.id;
+                        dev_priv->cfb_y = work->crtc->y;
+                }
+                dev_priv->fbc_work = NULL;
+        }
+        mutex_unlock(&dev->struct_mutex);
+        kfree(work);
+}
+static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
+{
+        if (dev_priv->fbc_work == NULL)
+                return;
+        DRM_DEBUG_KMS("cancelling pending FBC enable\n");
+        /* Synchronisation is provided by struct_mutex and checking of
+         * dev_priv->fbc_work, so we can perform the cancellation
+         * entirely asynchronously.
+         */
+        if (cancel_delayed_work(&dev_priv->fbc_work->work))
+                /* tasklet was killed before being run, clean up */
+                kfree(dev_priv->fbc_work);
+        /* Mark the work as no longer wanted so that if it does
+         * wake-up (because the work was already running and waiting
+         * for our mutex), it will discover that is no longer
+         * necessary to run.
+         */
+        dev_priv->fbc_work = NULL;
+}
+static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+        struct intel_fbc_work *work;
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
        if (!dev_priv->display.enable_fbc)
                return;
-        dev_priv->display.enable_fbc(crtc, interval);
+        intel_cancel_fbc_work(dev_priv);
+        work = kzalloc(sizeof *work, GFP_KERNEL);
+        if (work == NULL) {
+                dev_priv->display.enable_fbc(crtc, interval);
+                return;
+        }
+        work->crtc = crtc;
+        work->fb = crtc->fb;
+        work->interval = interval;
+        INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
+        dev_priv->fbc_work = work;
+        DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
+        /* Delay the actual enabling to let pageflipping cease and the
+         * display to settle before starting the compression. Note that
+         * this delay also serves a second purpose: it allows for a
+         * vblank to pass after disabling the FBC before we attempt
+         * to modify the control registers.
+         *
+         * A more complicated solution would involve tracking vblanks
+         * following the termination of the page-flipping sequence
+         * and indeed performing the enable as a co-routine and not
+         * waiting synchronously upon the vblank.
+         */
+        schedule_delayed_work(&work->work, msecs_to_jiffies(50));
 }
 void intel_disable_fbc(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
+        intel_cancel_fbc_work(dev_priv);
        if (!dev_priv->display.disable_fbc)
                return;
        dev_priv->display.disable_fbc(dev);
+        dev_priv->cfb_plane = -1;
 }
 /**
@@ -1760,8 +1796,13 @@ static void intel_update_fbc(struct drm_device *dev)
                dev_priv->no_fbc_reason = FBC_BAD_PLANE;
                goto out_disable;
        }
-        if (obj->tiling_mode != I915_TILING_X) {
-                DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n");
+        /* The use of a CPU fence is mandatory in order to detect writes
+         * by the CPU to the scanout and trigger updates to the FBC.
+         */
+        if (obj->tiling_mode != I915_TILING_X ||
+            obj->fence_reg == I915_FENCE_REG_NONE) {
+                DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
                dev_priv->no_fbc_reason = FBC_NOT_TILED;
                goto out_disable;
        }
@@ -1770,6 +1811,44 @@ static void intel_update_fbc(struct drm_device *dev)
        if (in_dbg_master())
                goto out_disable;
+        /* If the scanout has not changed, don't modify the FBC settings.
+         * Note that we make the fundamental assumption that the fb->obj
+         * cannot be unpinned (and have its GTT offset and fence revoked)
+         * without first being decoupled from the scanout and FBC disabled.
+         */
+        if (dev_priv->cfb_plane == intel_crtc->plane &&
+            dev_priv->cfb_fb == fb->base.id &&
+            dev_priv->cfb_y == crtc->y)
+                return;
+        if (intel_fbc_enabled(dev)) {
+                /* We update FBC along two paths, after changing fb/crtc
+                 * configuration (modeswitching) and after page-flipping
+                 * finishes. For the latter, we know that not only did
+                 * we disable the FBC at the start of the page-flip
+                 * sequence, but also more than one vblank has passed.
+                 *
+                 * For the former case of modeswitching, it is possible
+                 * to switch between two FBC valid configurations
+                 * instantaneously so we do need to disable the FBC
+                 * before we can modify its control registers. We also
+                 * have to wait for the next vblank for that to take
+                 * effect. However, since we delay enabling FBC we can
+                 * assume that a vblank has passed since disabling and
+                 * that we can safely alter the registers in the deferred
+                 * callback.
+                 *
+                 * In the scenario that we go from a valid to invalid
+                 * and then back to valid FBC configuration we have
+                 * no strict enforcement that a vblank occurred since
+                 * disabling the FBC. However, along all current pipe
+                 * disabling paths we do need to wait for a vblank at
+                 * some point. And we wait before enabling FBC anyway.
+                 */
+                DRM_DEBUG_KMS("disabling active FBC for update\n");
+                intel_disable_fbc(dev);
+        }
        intel_enable_fbc(crtc, 500);
        return;
@@ -1812,14 +1891,10 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
        }
        dev_priv->mm.interruptible = false;
-        ret = i915_gem_object_pin(obj, alignment, true);
+        ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined);
        if (ret)
                goto err_interruptible;
-        ret = i915_gem_object_set_to_display_plane(obj, pipelined);
-        if (ret)
-                goto err_unpin;
        /* Install a fence for tiled scan-out. Pre-i965 always needs a
         * fence, whereas 965+ only requires a fence if using
         * framebuffer compression.  For simplicity, we always install
@@ -1841,10 +1916,8 @@ err_interruptible:
        return ret;
 }
-/* Assume fb object is pinned & idle & fenced and just update base pointers */
+static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-static int
+                             int x, int y)
-intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-                           int x, int y, enum mode_set_atomic state)
 {
        struct drm_device *dev = crtc->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1887,7 +1960,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
                break;
        default:
-                DRM_ERROR("Unknown color depth\n");
+                DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
                return -EINVAL;
        }
        if (INTEL_INFO(dev)->gen >= 4) {
@@ -1897,10 +1970,6 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                        dspcntr &= ~DISPPLANE_TILED;
        }
-        if (HAS_PCH_SPLIT(dev))
-                /* must disable */
-                dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
        I915_WRITE(reg, dspcntr);
        Start = obj->gtt_offset;
@@ -1917,6 +1986,99 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                I915_WRITE(DSPADDR(plane), Start + Offset);
        POSTING_READ(reg);
+        return 0;
+}
+static int ironlake_update_plane(struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb, int x, int y)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_framebuffer *intel_fb;
+        struct drm_i915_gem_object *obj;
+        int plane = intel_crtc->plane;
+        unsigned long Start, Offset;
+        u32 dspcntr;
+        u32 reg;
+        switch (plane) {
+        case 0:
+        case 1:
+                break;
+        default:
+                DRM_ERROR("Can't update plane %d in SAREA\n", plane);
+                return -EINVAL;
+        }
+        intel_fb = to_intel_framebuffer(fb);
+        obj = intel_fb->obj;
+        reg = DSPCNTR(plane);
+        dspcntr = I915_READ(reg);
+        /* Mask out pixel format bits in case we change it */
+        dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
+        switch (fb->bits_per_pixel) {
+        case 8:
+                dspcntr |= DISPPLANE_8BPP;
+                break;
+        case 16:
+                if (fb->depth != 16)
+                        return -EINVAL;
+                dspcntr |= DISPPLANE_16BPP;
+                break;
+        case 24:
+        case 32:
+                if (fb->depth == 24)
+                        dspcntr |= DISPPLANE_32BPP_NO_ALPHA;
+                else if (fb->depth == 30)
+                        dspcntr |= DISPPLANE_32BPP_30BIT_NO_ALPHA;
+                else
+                        return -EINVAL;
+                break;
+        default:
+                DRM_ERROR("Unknown color depth %d\n", fb->bits_per_pixel);
+                return -EINVAL;
+        }
+        if (obj->tiling_mode != I915_TILING_NONE)
+                dspcntr |= DISPPLANE_TILED;
+        else
+                dspcntr &= ~DISPPLANE_TILED;
+        /* must disable */
+        dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
+        I915_WRITE(reg, dspcntr);
+        Start = obj->gtt_offset;
+        Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
+        DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
+                      Start, Offset, x, y, fb->pitch);
+        I915_WRITE(DSPSTRIDE(plane), fb->pitch);
+        I915_WRITE(DSPSURF(plane), Start);
+        I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
+        I915_WRITE(DSPADDR(plane), Offset);
+        POSTING_READ(reg);
+        return 0;
+}
+/* Assume fb object is pinned & idle & fenced and just update base pointers */
+static int
+intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                           int x, int y, enum mode_set_atomic state)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+        ret = dev_priv->display.update_plane(crtc, fb, x, y);
+        if (ret)
+                return ret;
        intel_update_fbc(dev);
        intel_increase_pllclock(crtc);
@@ -1934,7 +2096,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
        /* no fb bound */
        if (!crtc->fb) {
-                DRM_DEBUG_KMS("No FB bound\n");
+                DRM_ERROR("No FB bound\n");
                return 0;
        }
@@ -1943,6 +2105,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
        case 1:
                break;
        default:
+                DRM_ERROR("no plane for crtc\n");
                return -EINVAL;
        }
@@ -1952,6 +2115,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                                         NULL);
        if (ret != 0) {
                mutex_unlock(&dev->struct_mutex);
+                DRM_ERROR("pin & fence failed\n");
                return ret;
        }
@@ -1971,7 +2135,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 * This should only fail upon a hung GPU, in which case we
                 * can safely continue.
                 */
-                ret = i915_gem_object_flush_gpu(obj);
+                ret = i915_gem_object_finish_gpu(obj);
                (void) ret;
        }
@@ -1980,6 +2144,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
        if (ret) {
                i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
                mutex_unlock(&dev->struct_mutex);
+                DRM_ERROR("failed to update base address\n");
                return ret;
        }
@@ -2622,6 +2787,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
        /* For PCH DP, enable TRANS_DP_CTL */
        if (HAS_PCH_CPT(dev) &&
            intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
+                u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) >> 5;
                reg = TRANS_DP_CTL(pipe);
                temp = I915_READ(reg);
                temp &= ~(TRANS_DP_PORT_SEL_MASK |
@@ -2629,7 +2795,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
                          TRANS_DP_BPC_MASK);
                temp |= (TRANS_DP_OUTPUT_ENABLE |
                         TRANS_DP_ENH_FRAMING);
-                temp |= TRANS_DP_8BPC;
+                temp |= bpc << 9; /* same format but at 11:9 */
                if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
                        temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
@@ -2736,9 +2902,8 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
        intel_disable_plane(dev_priv, plane, pipe);
-        if (dev_priv->cfb_plane == plane &&
+        if (dev_priv->cfb_plane == plane)
-            dev_priv->display.disable_fbc)
+                intel_disable_fbc(dev);
-                dev_priv->display.disable_fbc(dev);
        intel_disable_pipe(dev_priv, pipe);
@@ -2902,9 +3067,8 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
        intel_crtc_dpms_overlay(intel_crtc, false);
        intel_crtc_update_cursor(crtc, false);
-        if (dev_priv->cfb_plane == plane &&
+        if (dev_priv->cfb_plane == plane)
-            dev_priv->display.disable_fbc)
+                intel_disable_fbc(dev);
-                dev_priv->display.disable_fbc(dev);
        intel_disable_plane(dev_priv, plane, pipe);
        intel_disable_pipe(dev_priv, pipe);
@@ -4313,6 +4477,133 @@ static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
                && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
 }
+/**
+ * intel_choose_pipe_bpp_dither - figure out what color depth the pipe should send
+ * @crtc: CRTC structure
+ *
+ * A pipe may be connected to one or more outputs.  Based on the depth of the
+ * attached framebuffer, choose a good color depth to use on the pipe.
+ *
+ * If possible, match the pipe depth to the fb depth.  In some cases, this
+ * isn't ideal, because the connected output supports a lesser or restricted
+ * set of depths.  Resolve that here:
+ *    LVDS typically supports only 6bpc, so clamp down in that case
+ *    HDMI supports only 8bpc or 12bpc, so clamp to 8bpc with dither for 10bpc
+ *    Displays may support a restricted set as well, check EDID and clamp as
+ *      appropriate.
+ *
+ * RETURNS:
+ * Dithering requirement (i.e. false if display bpc and pipe bpc match,
+ * true if they don't match).
+ */
+static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
+                                         unsigned int *pipe_bpp)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_encoder *encoder;
+        struct drm_connector *connector;
+        unsigned int display_bpc = UINT_MAX, bpc;
+        /* Walk the encoders & connectors on this crtc, get min bpc */
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+                struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
+                if (encoder->crtc != crtc)
+                        continue;
+                if (intel_encoder->type == INTEL_OUTPUT_LVDS) {
+                        unsigned int lvds_bpc;
+                        if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) ==
+                            LVDS_A3_POWER_UP)
+                                lvds_bpc = 8;
+                        else
+                                lvds_bpc = 6;
+                        if (lvds_bpc < display_bpc) {
+                                DRM_DEBUG_DRIVER("clamping display bpc (was %d) to LVDS (%d)\n", display_bpc, lvds_bpc);
+                                display_bpc = lvds_bpc;
+                        }
+                        continue;
+                }
+                if (intel_encoder->type == INTEL_OUTPUT_EDP) {
+                        /* Use VBT settings if we have an eDP panel */
+                        unsigned int edp_bpc = dev_priv->edp.bpp / 3;
+                        if (edp_bpc < display_bpc) {
+                                DRM_DEBUG_DRIVER("clamping display bpc (was %d) to eDP (%d)\n", display_bpc, edp_bpc);
+                                display_bpc = edp_bpc;
+                        }
+                        continue;
+                }
+                /* Not one of the known troublemakers, check the EDID */
+                list_for_each_entry(connector, &dev->mode_config.connector_list,
+                                    head) {
+                        if (connector->encoder != encoder)
+                                continue;
+                        if (connector->display_info.bpc < display_bpc) {
+                                DRM_DEBUG_DRIVER("clamping display bpc (was %d) to EDID reported max of %d\n", display_bpc, connector->display_info.bpc);
+                                display_bpc = connector->display_info.bpc;
+                        }
+                }
+                /*
+                 * HDMI is either 12 or 8, so if the display lets 10bpc sneak
+                 * through, clamp it down.  (Note: >12bpc will be caught below.)
+                 */
+                if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
+                        if (display_bpc > 8 && display_bpc < 12) {
+                                DRM_DEBUG_DRIVER("forcing bpc to 12 for HDMI\n");
+                                display_bpc = 12;
+                        } else {
+                                DRM_DEBUG_DRIVER("forcing bpc to 8 for HDMI\n");
+                                display_bpc = 8;
+                        }
+                }
+        }
+        /*
+         * We could just drive the pipe at the highest bpc all the time and
+         * enable dithering as needed, but that costs bandwidth.  So choose
+         * the minimum value that expresses the full color range of the fb but
+         * also stays within the max display bpc discovered above.
+         */
+        switch (crtc->fb->depth) {
+        case 8:
+                bpc = 8; /* since we go through a colormap */
+                break;
+        case 15:
+        case 16:
+                bpc = 6; /* min is 18bpp */
+                break;
+        case 24:
+                bpc = min((unsigned int)8, display_bpc);
+                break;
+        case 30:
+                bpc = min((unsigned int)10, display_bpc);
+                break;
+        case 48:
+                bpc = min((unsigned int)12, display_bpc);
+                break;
+        default:
+                DRM_DEBUG("unsupported depth, assuming 24 bits\n");
+                bpc = min((unsigned int)8, display_bpc);
+                break;
+        }
+        DRM_DEBUG_DRIVER("setting pipe bpc to %d (max display bpc %d)\n",
+                         bpc, display_bpc);
+        *pipe_bpp = bpc * 3;
+        return display_bpc != bpc;
+}
 static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
                              struct drm_display_mode *mode,
                              struct drm_display_mode *adjusted_mode,
@@ -4725,7 +5016,9 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
        struct fdi_m_n m_n = {0};
        u32 temp;
        u32 lvds_sync = 0;
-        int target_clock, pixel_multiplier, lane, link_bw, bpp, factor;
+        int target_clock, pixel_multiplier, lane, link_bw, factor;
+        unsigned int pipe_bpp;
+        bool dither;
        list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
                if (encoder->base.crtc != crtc)
@@ -4852,56 +5145,37 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
        /* determine panel color depth */
        temp = I915_READ(PIPECONF(pipe));
        temp &= ~PIPE_BPC_MASK;
-        if (is_lvds) {
+        dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp);
-                /* the BPC will be 6 if it is 18-bit LVDS panel */
+        switch (pipe_bpp) {
-                if ((I915_READ(PCH_LVDS) & LVDS_A3_POWER_MASK) == LVDS_A3_POWER_UP)
+        case 18:
-                        temp |= PIPE_8BPC;
+                temp |= PIPE_6BPC;
-                else
-                        temp |= PIPE_6BPC;
-        } else if (has_edp_encoder) {
-                switch (dev_priv->edp.bpp/3) {
-                case 8:
-                        temp |= PIPE_8BPC;
-                        break;
-                case 10:
-                        temp |= PIPE_10BPC;
-                        break;
-                case 6:
-                        temp |= PIPE_6BPC;
-                        break;
-                case 12:
-                        temp |= PIPE_12BPC;
-                        break;
-                }
-        } else
-                temp |= PIPE_8BPC;
-        I915_WRITE(PIPECONF(pipe), temp);
-        switch (temp & PIPE_BPC_MASK) {
-        case PIPE_8BPC:
-                bpp = 24;
                break;
-        case PIPE_10BPC:
+        case 24:
-                bpp = 30;
+                temp |= PIPE_8BPC;
                break;
-        case PIPE_6BPC:
+        case 30:
-                bpp = 18;
+                temp |= PIPE_10BPC;
                break;
-        case PIPE_12BPC:
+        case 36:
-                bpp = 36;
+                temp |= PIPE_12BPC;
                break;
        default:
-                DRM_ERROR("unknown pipe bpc value\n");
+                WARN(1, "intel_choose_pipe_bpp returned invalid value\n");
-                bpp = 24;
+                temp |= PIPE_8BPC;
+                pipe_bpp = 24;
+                break;
        }
+        intel_crtc->bpp = pipe_bpp;
+        I915_WRITE(PIPECONF(pipe), temp);
        if (!lane) {
                /*
                 * Account for spread spectrum to avoid
                 * oversubscribing the link. Max center spread
                 * is 2.5%; use 5% for safety's sake.
                 */
-                u32 bps = target_clock * bpp * 21 / 20;
+                u32 bps = target_clock * intel_crtc->bpp * 21 / 20;
                lane = bps / (link_bw * 8) + 1;
        }
@@ -4909,7 +5183,8 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
        if (pixel_multiplier > 1)
                link_bw *= pixel_multiplier;
-        ironlake_compute_m_n(bpp, lane, target_clock, link_bw, &m_n);
+        ironlake_compute_m_n(intel_crtc->bpp, lane, target_clock, link_bw,
+                             &m_n);
        /* Ironlake: try to setup display ref clock before DPLL
         * enabling. This is only under driver's control after
@@ -5112,14 +5387,12 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
                I915_WRITE(PCH_LVDS, temp);
        }
-        /* set the dithering flag and clear for anything other than a panel. */
        pipeconf &= ~PIPECONF_DITHER_EN;
        pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
-        if (dev_priv->lvds_dither && (is_lvds || has_edp_encoder)) {
+        if ((is_lvds && dev_priv->lvds_dither) || dither) {
                pipeconf |= PIPECONF_DITHER_EN;
                pipeconf |= PIPECONF_DITHER_TYPE_ST1;
        }
        if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
                intel_dp_set_m_n(crtc, mode, adjusted_mode);
        } else {
@@ -5439,21 +5712,15 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                        goto fail_locked;
                }
-                ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
+                ret = i915_gem_object_pin_to_display_plane(obj, 0, NULL);
-                if (ret) {
-                        DRM_ERROR("failed to pin cursor bo\n");
-                        goto fail_locked;
-                }
-                ret = i915_gem_object_set_to_gtt_domain(obj, 0);
                if (ret) {
                        DRM_ERROR("failed to move cursor bo into the GTT\n");
-                        goto fail_unpin;
+                        goto fail_locked;
                }
                ret = i915_gem_object_put_fence(obj);
                if (ret) {
-                        DRM_ERROR("failed to move cursor bo into the GTT\n");
+                        DRM_ERROR("failed to release fence for cursor");
                        goto fail_unpin;
                }
@@ -6156,6 +6423,7 @@ static void intel_unpin_work_fn(struct work_struct *__work)
        drm_gem_object_unreference(&work->pending_flip_obj->base);
        drm_gem_object_unreference(&work->old_fb_obj->base);
+        intel_update_fbc(work->dev);
        mutex_unlock(&work->dev->struct_mutex);
        kfree(work);
 }
@@ -6520,6 +6788,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
        if (ret)
                goto cleanup_pending;
+        intel_disable_fbc(dev);
        mutex_unlock(&dev->struct_mutex);
        trace_i915_flip_request(intel_crtc->plane, obj);
@@ -6648,6 +6917,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
        intel_crtc_reset(&intel_crtc->base);
        intel_crtc->active = true; /* force the pipe off on setup_init_config */
+        intel_crtc->bpp = 24; /* default for pre-Ironlake */
        if (HAS_PCH_SPLIT(dev)) {
                intel_helper_funcs.prepare = ironlake_crtc_prepare;
@@ -6874,6 +7144,11 @@ int intel_framebuffer_init(struct drm_device *dev,
        switch (mode_cmd->bpp) {
        case 8:
        case 16:
+                /* Only pre-ILK can handle 5:5:5 */
+                if (mode_cmd->depth == 15 && !HAS_PCH_SPLIT(dev))
+                        return -EINVAL;
+                break;
        case 24:
        case 32:
                break;
@@ -7288,6 +7563,59 @@ void gen6_enable_rps(struct drm_i915_private *dev_priv)
        mutex_unlock(&dev_priv->dev->struct_mutex);
 }
+void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
+{
+        int min_freq = 15;
+        int gpu_freq, ia_freq, max_ia_freq;
+        int scaling_factor = 180;
+        max_ia_freq = cpufreq_quick_get_max(0);
+        /*
+         * Default to measured freq if none found, PCU will ensure we don't go
+         * over
+         */
+        if (!max_ia_freq)
+                max_ia_freq = tsc_khz;
+        /* Convert from kHz to MHz */
+        max_ia_freq /= 1000;
+        mutex_lock(&dev_priv->dev->struct_mutex);
+        /*
+         * For each potential GPU frequency, load a ring frequency we'd like
+         * to use for memory access.  We do this by specifying the IA frequency
+         * the PCU should use as a reference to determine the ring frequency.
+         */
+        for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
+             gpu_freq--) {
+                int diff = dev_priv->max_delay - gpu_freq;
+                /*
+                 * For GPU frequencies less than 750MHz, just use the lowest
+                 * ring freq.
+                 */
+                if (gpu_freq < min_freq)
+                        ia_freq = 800;
+                else
+                        ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
+                ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
+                I915_WRITE(GEN6_PCODE_DATA,
+                           (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
+                           gpu_freq);
+                I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
+                           GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
+                if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
+                              GEN6_PCODE_READY) == 0, 10)) {
+                        DRM_ERROR("pcode write of freq table timed out\n");
+                        continue;
+                }
+        }
+        mutex_unlock(&dev_priv->dev->struct_mutex);
+}
 static void ironlake_init_clock_gating(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
@@ -7644,9 +7972,11 @@ static void intel_init_display(struct drm_device *dev)
        if (HAS_PCH_SPLIT(dev)) {
                dev_priv->display.dpms = ironlake_crtc_dpms;
                dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
+                dev_priv->display.update_plane = ironlake_update_plane;
        } else {
                dev_priv->display.dpms = i9xx_crtc_dpms;
                dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
+                dev_priv->display.update_plane = i9xx_update_plane;
        }
        if (I915_HAS_FBC(dev)) {
@@ -7943,8 +8273,10 @@ void intel_modeset_init(struct drm_device *dev)
                intel_init_emon(dev);
        }
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev) || IS_GEN7(dev)) {
                gen6_enable_rps(dev_priv);
+                gen6_update_ring_freq(dev_priv);
+        }
        INIT_WORK(&dev_priv->idle_work, intel_idle_update);
        setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
@@ -7980,12 +8312,11 @@ void intel_modeset_cleanup(struct drm_device *dev)
                intel_increase_pllclock(crtc);
        }
-        if (dev_priv->display.disable_fbc)
+        intel_disable_fbc(dev);
-                dev_priv->display.disable_fbc(dev);
        if (IS_IRONLAKE_M(dev))
                ironlake_disable_drps(dev);
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev) || IS_GEN7(dev))
                gen6_disable_rps(dev);
        if (IS_IRONLAKE_M(dev))
@@ -7998,6 +8329,9 @@ void intel_modeset_cleanup(struct drm_device *dev)
        drm_irq_uninstall(dev);
        cancel_work_sync(&dev_priv->hotplug_work);
+        /* flush any delayed tasks or pending work */
+        flush_scheduled_work();
        /* Shut off idle work before the crtcs get freed. */
        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                intel_crtc = to_intel_crtc(crtc);
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 1c3a36feaf69..52b79d9d5d2d 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -178,12 +178,14 @@ intel_dp_link_clock(uint8_t link_bw)
 static int
 intel_dp_link_required(struct drm_device *dev, struct intel_dp *intel_dp, int pixel_clock)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = intel_dp->base.base.crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int bpp = 24;
-        if (is_edp(intel_dp))
+        if (intel_crtc)
-                return (pixel_clock * dev_priv->edp.bpp + 7) / 8;
+                bpp = intel_crtc->bpp;
-        else
-                return pixel_clock * 3;
+        return (pixel_clock * bpp + 7) / 8;
 }
 static int
@@ -681,7 +683,7 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
        struct drm_encoder *encoder;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int lane_count = 4, bpp = 24;
+        int lane_count = 4;
        struct intel_dp_m_n m_n;
        int pipe = intel_crtc->pipe;
@@ -700,7 +702,6 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
                        break;
                } else if (is_edp(intel_dp)) {
                        lane_count = dev_priv->edp.lanes;
-                        bpp = dev_priv->edp.bpp;
                        break;
                }
        }
@@ -710,7 +711,7 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
         * the number of bytes_per_pixel post-LUT, which we always
         * set up for 8-bits of R/G/B, or 3 bytes total.
         */
-        intel_dp_compute_m_n(bpp, lane_count,
+        intel_dp_compute_m_n(intel_crtc->bpp, lane_count,
                             mode->clock, adjusted_mode->clock, &m_n);
        if (HAS_PCH_SPLIT(dev)) {
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 9ffa61eb4d7e..6e990f9760ef 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -170,6 +170,7 @@ struct intel_crtc {
        int16_t cursor_x, cursor_y;
        int16_t cursor_width, cursor_height;
        bool cursor_visible;
+        unsigned int bpp;
 };
 #define to_intel_crtc(x) container_of(x, struct intel_crtc, base)
@@ -233,6 +234,13 @@ struct intel_unpin_work {
        bool enable_stall_check;
 };
+struct intel_fbc_work {
+        struct delayed_work work;
+        struct drm_crtc *crtc;
+        struct drm_framebuffer *fb;
+        int interval;
+};
 int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
 extern bool intel_ddc_probe(struct intel_encoder *intel_encoder, int ddc_bus);
@@ -317,6 +325,7 @@ extern void intel_enable_clock_gating(struct drm_device *dev);
 extern void ironlake_enable_drps(struct drm_device *dev);
 extern void ironlake_disable_drps(struct drm_device *dev);
 extern void gen6_enable_rps(struct drm_i915_private *dev_priv);
+extern void gen6_update_ring_freq(struct drm_i915_private *dev_priv);
 extern void gen6_disable_rps(struct drm_device *dev);
 extern void intel_init_emon(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index aa0a8e83142e..1ed8e6903915 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -124,12 +124,18 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
        u32 sdvox;
        sdvox = SDVO_ENCODING_HDMI | SDVO_BORDER_ENABLE;
-        sdvox |= intel_hdmi->color_range;
+        if (!HAS_PCH_SPLIT(dev))
+                sdvox |= intel_hdmi->color_range;
        if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
                sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
        if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
                sdvox |= SDVO_HSYNC_ACTIVE_HIGH;
+        if (intel_crtc->bpp > 24)
+                sdvox |= COLOR_FORMAT_12bpc;
+        else
+                sdvox |= COLOR_FORMAT_8bpc;
        /* Required on CPT */
        if (intel_hdmi->has_hdmi_sink && HAS_PCH_CPT(dev))
                sdvox |= HDMI_MODE_SELECT;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index d2c710422908..b7c5ddb564d1 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -297,19 +297,26 @@ static int intel_opregion_video_event(struct notifier_block *nb,
        /* The only video events relevant to opregion are 0x80. These indicate
           either a docking event, lid switch or display switch request. In
           Linux, these are handled by the dock, button and video drivers.
-           We might want to fix the video driver to be opregion-aware in
+        */
-           future, but right now we just indicate to the firmware that the
-           request has been handled */
        struct opregion_acpi *acpi;
+        struct acpi_bus_event *event = data;
+        int ret = NOTIFY_OK;
+        if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0)
+                return NOTIFY_DONE;
        if (!system_opregion)
                return NOTIFY_DONE;
        acpi = system_opregion->acpi;
+        if (event->type == 0x80 && !(acpi->cevt & 0x1))
+                ret = NOTIFY_BAD;
        acpi->csts = 0;
-        return NOTIFY_OK;
+        return ret;
 }
 static struct notifier_block intel_opregion_notifier = {
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 9e2959bc91cd..d36038086826 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -773,14 +773,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
        if (ret != 0)
                return ret;
-        ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true);
+        ret = i915_gem_object_pin_to_display_plane(new_bo, 0, NULL);
        if (ret != 0)
                return ret;
-        ret = i915_gem_object_set_to_gtt_domain(new_bo, 0);
-        if (ret != 0)
-                goto out_unpin;
        ret = i915_gem_object_put_fence(new_bo);
        if (ret)
                goto out_unpin;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 1f61fc7b754c..47b9b2777038 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -236,7 +236,8 @@ init_pipe_control(struct intel_ring_buffer *ring)
                ret = -ENOMEM;
                goto err;
        }
-        obj->cache_level = I915_CACHE_LLC;
+        i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
        ret = i915_gem_object_pin(obj, 4096, true);
        if (ret)
@@ -776,7 +777,8 @@ static int init_status_page(struct intel_ring_buffer *ring)
                ret = -ENOMEM;
                goto err;
        }
-        obj->cache_level = I915_CACHE_LLC;
+        i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
        ret = i915_gem_object_pin(obj, 4096, true);
        if (ret != 0) {
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 113e4e7264cd..210d570fd516 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1236,6 +1236,8 @@ intel_tv_detect_type (struct intel_tv *intel_tv,
                      struct drm_connector *connector)
 {
        struct drm_encoder *encoder = &intel_tv->base.base;
+        struct drm_crtc *crtc = encoder->crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        struct drm_device *dev = encoder->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        unsigned long irqflags;
@@ -1258,6 +1260,10 @@ intel_tv_detect_type (struct intel_tv *intel_tv,
        /* Poll for TV detection */
        tv_ctl &= ~(TV_ENC_ENABLE | TV_TEST_MODE_MASK);
        tv_ctl |= TV_TEST_MODE_MONITOR_DETECT;
+        if (intel_crtc->pipe == 1)
+                tv_ctl |= TV_ENC_PIPEB_SELECT;
+        else
+                tv_ctl &= ~TV_ENC_PIPEB_SELECT;
        tv_dac &= ~(TVDAC_SENSE_MASK | DAC_A_MASK | DAC_B_MASK | DAC_C_MASK);
        tv_dac |= (TVDAC_STATE_CHG_EN |
@@ -1277,26 +1283,26 @@ intel_tv_detect_type (struct intel_tv *intel_tv,
                              to_intel_crtc(intel_tv->base.base.crtc)->pipe);
        type = -1;
-        if (wait_for((tv_dac = I915_READ(TV_DAC)) & TVDAC_STATE_CHG, 20) == 0) {
+        tv_dac = I915_READ(TV_DAC);
-                DRM_DEBUG_KMS("TV detected: %x, %x\n", tv_ctl, tv_dac);
+        DRM_DEBUG_KMS("TV detected: %x, %x\n", tv_ctl, tv_dac);
-                /*
+        /*
-                 *  A B C
+         *  A B C
-                 *  0 1 1 Composite
+         *  0 1 1 Composite
-                 *  1 0 X svideo
+         *  1 0 X svideo
-                 *  0 0 0 Component
+         *  0 0 0 Component
-                 */
+         */
-                if ((tv_dac & TVDAC_SENSE_MASK) == (TVDAC_B_SENSE | TVDAC_C_SENSE)) {
+        if ((tv_dac & TVDAC_SENSE_MASK) == (TVDAC_B_SENSE | TVDAC_C_SENSE)) {
-                        DRM_DEBUG_KMS("Detected Composite TV connection\n");
+                DRM_DEBUG_KMS("Detected Composite TV connection\n");
-                        type = DRM_MODE_CONNECTOR_Composite;
+                type = DRM_MODE_CONNECTOR_Composite;
-                } else if ((tv_dac & (TVDAC_A_SENSE|TVDAC_B_SENSE)) == TVDAC_A_SENSE) {
+        } else if ((tv_dac & (TVDAC_A_SENSE|TVDAC_B_SENSE)) == TVDAC_A_SENSE) {
-                        DRM_DEBUG_KMS("Detected S-Video TV connection\n");
+                DRM_DEBUG_KMS("Detected S-Video TV connection\n");
-                        type = DRM_MODE_CONNECTOR_SVIDEO;
+                type = DRM_MODE_CONNECTOR_SVIDEO;
-                } else if ((tv_dac & TVDAC_SENSE_MASK) == 0) {
+        } else if ((tv_dac & TVDAC_SENSE_MASK) == 0) {
-                        DRM_DEBUG_KMS("Detected Component TV connection\n");
+                DRM_DEBUG_KMS("Detected Component TV connection\n");
-                        type = DRM_MODE_CONNECTOR_Component;
+                type = DRM_MODE_CONNECTOR_Component;
-                } else {
+        } else {
-                        DRM_DEBUG_KMS("Unrecognised TV connection\n");
+                DRM_DEBUG_KMS("Unrecognised TV connection\n");
-                }
+                type = -1;
        }
        I915_WRITE(TV_DAC, save_tv_dac & ~TVDAC_STATE_CHG_EN);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 729d5fd7c88d..1aa73d3957e1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -135,13 +135,14 @@ static void load_vbios_pramin(struct drm_device *dev, uint8_t *data)
        int i;
        if (dev_priv->card_type >= NV_50) {
-                uint32_t vbios_vram = (nv_rd32(dev, 0x619f04) & ~0xff) << 8;
+                u64 addr = (u64)(nv_rd32(dev, 0x619f04) & 0xffffff00) << 8;
+                if (!addr) {
-                if (!vbios_vram)
+                        addr  = (u64)nv_rd32(dev, 0x1700) << 16;
-                        vbios_vram = (nv_rd32(dev, 0x1700) << 16) + 0xf0000;
+                        addr += 0xf0000;
+                }
                old_bar0_pramin = nv_rd32(dev, 0x1700);
-                nv_wr32(dev, 0x1700, vbios_vram >> 16);
+                nv_wr32(dev, 0x1700, addr >> 16);
        }
        /* bail if no rom signature */
@@ -5186,7 +5187,7 @@ static int parse_bit_A_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
        load_table_ptr = ROM16(bios->data[bitentry->offset]);
        if (load_table_ptr == 0x0) {
-                NV_ERROR(dev, "Pointer to BIT loadval table invalid\n");
+                NV_DEBUG(dev, "Pointer to BIT loadval table invalid\n");
                return -EINVAL;
        }
@@ -6377,6 +6378,37 @@ apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
                }
        }
+        /* Some other twisted XFX board (rhbz#694914)
+         *
+         * The DVI/VGA encoder combo that's supposed to represent the
+         * DVI-I connector actually point at two different ones, and
+         * the HDMI connector ends up paired with the VGA instead.
+         *
+         * Connector table is missing anything for VGA at all, pointing it
+         * an invalid conntab entry 2 so we figure it out ourself.
+         */
+        if (nv_match_device(dev, 0x0615, 0x1682, 0x2605)) {
+                if (idx == 0) {
+                        *conn = 0x02002300; /* VGA, connector 2 */
+                        *conf = 0x00000028;
+                } else
+                if (idx == 1) {
+                        *conn = 0x01010312; /* DVI, connector 0 */
+                        *conf = 0x00020030;
+                } else
+                if (idx == 2) {
+                        *conn = 0x04020310; /* VGA, connector 0 */
+                        *conf = 0x00000028;
+                } else
+                if (idx == 3) {
+                        *conn = 0x02021322; /* HDMI, connector 1 */
+                        *conf = 0x00020010;
+                } else {
+                        *conn = 0x0000000e; /* EOL */
+                        *conf = 0x00000000;
+                }
+        }
        return true;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 2ad49cbf7c8b..890d50e4d682 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -49,16 +49,12 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
                DRM_ERROR("bo %p still attached to GEM object\n", bo);
        nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
-        if (nvbo->vma.node) {
-                nouveau_vm_unmap(&nvbo->vma);
-                nouveau_vm_put(&nvbo->vma);
-        }
        kfree(nvbo);
 }
 static void
 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
-                       int *align, int *size, int *page_shift)
+                       int *align, int *size)
 {
        struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
@@ -82,67 +78,51 @@ nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
                        }
                }
        } else {
-                if (likely(dev_priv->chan_vm)) {
+                *size = roundup(*size, (1 << nvbo->page_shift));
-                        if (!(flags & TTM_PL_FLAG_TT) &&  *size > 256 * 1024)
+                *align = max((1 <<  nvbo->page_shift), *align);
-                                *page_shift = dev_priv->chan_vm->lpg_shift;
-                        else
-                                *page_shift = dev_priv->chan_vm->spg_shift;
-                } else {
-                        *page_shift = 12;
-                }
-                *size = roundup(*size, (1 << *page_shift));
-                *align = max((1 << *page_shift), *align);
        }
        *size = roundup(*size, PAGE_SIZE);
 }
 int
-nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
+nouveau_bo_new(struct drm_device *dev, int size, int align,
-               int size, int align, uint32_t flags, uint32_t tile_mode,
+               uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
-               uint32_t tile_flags, struct nouveau_bo **pnvbo)
+               struct nouveau_bo **pnvbo)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_bo *nvbo;
-        int ret = 0, page_shift = 0;
+        int ret;
        nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
        if (!nvbo)
                return -ENOMEM;
        INIT_LIST_HEAD(&nvbo->head);
        INIT_LIST_HEAD(&nvbo->entry);
+        INIT_LIST_HEAD(&nvbo->vma_list);
        nvbo->tile_mode = tile_mode;
        nvbo->tile_flags = tile_flags;
        nvbo->bo.bdev = &dev_priv->ttm.bdev;
-        nouveau_bo_fixup_align(nvbo, flags, &align, &size, &page_shift);
+        nvbo->page_shift = 12;
-        align >>= PAGE_SHIFT;
+        if (dev_priv->bar1_vm) {
+                if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
-        if (dev_priv->chan_vm) {
+                        nvbo->page_shift = dev_priv->bar1_vm->lpg_shift;
-                ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift,
-                                     NV_MEM_ACCESS_RW, &nvbo->vma);
-                if (ret) {
-                        kfree(nvbo);
-                        return ret;
-                }
        }
+        nouveau_bo_fixup_align(nvbo, flags, &align, &size);
        nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
        nouveau_bo_placement_set(nvbo, flags, 0);
-        nvbo->channel = chan;
        ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
-                          ttm_bo_type_device, &nvbo->placement, align, 0,
+                          ttm_bo_type_device, &nvbo->placement,
-                          false, NULL, size, nouveau_bo_del_ttm);
+                          align >> PAGE_SHIFT, 0, false, NULL, size,
+                          nouveau_bo_del_ttm);
        if (ret) {
                /* ttm will call nouveau_bo_del_ttm if it fails.. */
                return ret;
        }
-        nvbo->channel = NULL;
-        if (nvbo->vma.node)
-                nvbo->bo.offset = nvbo->vma.offset;
        *pnvbo = nvbo;
        return 0;
 }
@@ -312,8 +292,6 @@ nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
        if (ret)
                return ret;
-        if (nvbo->vma.node)
-                nvbo->bo.offset = nvbo->vma.offset;
        return 0;
 }
@@ -440,7 +418,6 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
                                     TTM_MEMTYPE_FLAG_CMA;
                        man->available_caching = TTM_PL_MASK_CACHING;
                        man->default_caching = TTM_PL_FLAG_CACHED;
-                        man->gpu_offset = dev_priv->gart_info.aper_base;
                        break;
                default:
                        NV_ERROR(dev, "Unknown GART type: %d\n",
@@ -501,19 +478,12 @@ static int
 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
                  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
-        struct nouveau_mem *old_node = old_mem->mm_node;
+        struct nouveau_mem *node = old_mem->mm_node;
-        struct nouveau_mem *new_node = new_mem->mm_node;
+        u64 src_offset = node->vma[0].offset;
-        struct nouveau_bo *nvbo = nouveau_bo(bo);
+        u64 dst_offset = node->vma[1].offset;
        u32 page_count = new_mem->num_pages;
-        u64 src_offset, dst_offset;
        int ret;
-        src_offset = old_node->tmp_vma.offset;
-        if (new_node->tmp_vma.node)
-                dst_offset = new_node->tmp_vma.offset;
-        else
-                dst_offset = nvbo->vma.offset;
        page_count = new_mem->num_pages;
        while (page_count) {
                int line_count = (page_count > 2047) ? 2047 : page_count;
@@ -547,19 +517,13 @@ static int
 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
                  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
-        struct nouveau_mem *old_node = old_mem->mm_node;
+        struct nouveau_mem *node = old_mem->mm_node;
-        struct nouveau_mem *new_node = new_mem->mm_node;
        struct nouveau_bo *nvbo = nouveau_bo(bo);
        u64 length = (new_mem->num_pages << PAGE_SHIFT);
-        u64 src_offset, dst_offset;
+        u64 src_offset = node->vma[0].offset;
+        u64 dst_offset = node->vma[1].offset;
        int ret;
-        src_offset = old_node->tmp_vma.offset;
-        if (new_node->tmp_vma.node)
-                dst_offset = new_node->tmp_vma.offset;
-        else
-                dst_offset = nvbo->vma.offset;
        while (length) {
                u32 amount, stride, height;
@@ -695,6 +659,27 @@ nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 }
 static int
+nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo,
+                   struct ttm_mem_reg *mem, struct nouveau_vma *vma)
+{
+        struct nouveau_mem *node = mem->mm_node;
+        int ret;
+        ret = nouveau_vm_get(chan->vm, mem->num_pages << PAGE_SHIFT,
+                             node->page_shift, NV_MEM_ACCESS_RO, vma);
+        if (ret)
+                return ret;
+        if (mem->mem_type == TTM_PL_VRAM)
+                nouveau_vm_map(vma, node);
+        else
+                nouveau_vm_map_sg(vma, 0, mem->num_pages << PAGE_SHIFT,
+                                  node, node->pages);
+        return 0;
+}
+static int
 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                     bool no_wait_reserve, bool no_wait_gpu,
                     struct ttm_mem_reg *new_mem)
@@ -711,31 +696,20 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
        }
-        /* create temporary vma for old memory, this will get cleaned
+        /* create temporary vmas for the transfer and attach them to the
-         * up after ttm destroys the ttm_mem_reg
+         * old nouveau_mem node, these will get cleaned up after ttm has
+         * destroyed the ttm_mem_reg
         */
        if (dev_priv->card_type >= NV_50) {
                struct nouveau_mem *node = old_mem->mm_node;
-                if (!node->tmp_vma.node) {
-                        u32 page_shift = nvbo->vma.node->type;
-                        if (old_mem->mem_type == TTM_PL_TT)
-                                page_shift = nvbo->vma.vm->spg_shift;
-                        ret = nouveau_vm_get(chan->vm,
-                                             old_mem->num_pages << PAGE_SHIFT,
-                                             page_shift, NV_MEM_ACCESS_RO,
-                                             &node->tmp_vma);
-                        if (ret)
-                                goto out;
-                }
-                if (old_mem->mem_type == TTM_PL_VRAM)
+                ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]);
-                        nouveau_vm_map(&node->tmp_vma, node);
+                if (ret)
-                else {
+                        goto out;
-                        nouveau_vm_map_sg(&node->tmp_vma, 0,
-                                          old_mem->num_pages << PAGE_SHIFT,
+                ret = nouveau_vma_getmap(chan, nvbo, new_mem, &node->vma[1]);
-                                          node, node->pages);
+                if (ret)
-                }
+                        goto out;
        }
        if (dev_priv->card_type < NV_50)
@@ -762,7 +736,6 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
                      bool no_wait_reserve, bool no_wait_gpu,
                      struct ttm_mem_reg *new_mem)
 {
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
        u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
        struct ttm_placement placement;
        struct ttm_mem_reg tmp_mem;
@@ -782,23 +755,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
        if (ret)
                goto out;
-        if (dev_priv->card_type >= NV_50) {
-                struct nouveau_bo *nvbo = nouveau_bo(bo);
-                struct nouveau_mem *node = tmp_mem.mm_node;
-                struct nouveau_vma *vma = &nvbo->vma;
-                if (vma->node->type != vma->vm->spg_shift)
-                        vma = &node->tmp_vma;
-                nouveau_vm_map_sg(vma, 0, tmp_mem.num_pages << PAGE_SHIFT,
-                                  node, node->pages);
-        }
        ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
-        if (dev_priv->card_type >= NV_50) {
-                struct nouveau_bo *nvbo = nouveau_bo(bo);
-                nouveau_vm_unmap(&nvbo->vma);
-        }
        if (ret)
                goto out;
@@ -844,30 +801,22 @@ out:
 static void
 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
 {
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
        struct nouveau_mem *node = new_mem->mm_node;
        struct nouveau_bo *nvbo = nouveau_bo(bo);
-        struct nouveau_vma *vma = &nvbo->vma;
+        struct nouveau_vma *vma;
-        struct nouveau_vm *vm = vma->vm;
+        list_for_each_entry(vma, &nvbo->vma_list, head) {
-        if (dev_priv->card_type < NV_50)
+                if (new_mem->mem_type == TTM_PL_VRAM) {
-                return;
+                        nouveau_vm_map(vma, new_mem->mm_node);
+                } else
-        switch (new_mem->mem_type) {
+                if (new_mem->mem_type == TTM_PL_TT &&
-        case TTM_PL_VRAM:
+                    nvbo->page_shift == vma->vm->spg_shift) {
-                nouveau_vm_map(vma, node);
+                        nouveau_vm_map_sg(vma, 0, new_mem->
-                break;
+                                          num_pages << PAGE_SHIFT,
-        case TTM_PL_TT:
+                                          node, node->pages);
-                if (vma->node->type != vm->spg_shift) {
+                } else {
                        nouveau_vm_unmap(vma);
-                        vma = &node->tmp_vma;
                }
-                nouveau_vm_map_sg(vma, 0, new_mem->num_pages << PAGE_SHIFT,
-                                  node, node->pages);
-                break;
-        default:
-                nouveau_vm_unmap(&nvbo->vma);
-                break;
        }
 }
@@ -1113,3 +1062,54 @@ struct ttm_bo_driver nouveau_bo_driver = {
        .io_mem_free = &nouveau_ttm_io_mem_free,
 };
+struct nouveau_vma *
+nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm)
+{
+        struct nouveau_vma *vma;
+        list_for_each_entry(vma, &nvbo->vma_list, head) {
+                if (vma->vm == vm)
+                        return vma;
+        }
+        return NULL;
+}
+int
+nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm,
+                   struct nouveau_vma *vma)
+{
+        const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
+        struct nouveau_mem *node = nvbo->bo.mem.mm_node;
+        int ret;
+        ret = nouveau_vm_get(vm, size, nvbo->page_shift,
+                             NV_MEM_ACCESS_RW, vma);
+        if (ret)
+                return ret;
+        if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
+                nouveau_vm_map(vma, nvbo->bo.mem.mm_node);
+        else
+        if (nvbo->bo.mem.mem_type == TTM_PL_TT)
+                nouveau_vm_map_sg(vma, 0, size, node, node->pages);
+        list_add_tail(&vma->head, &nvbo->vma_list);
+        vma->refcount = 1;
+        return 0;
+}
+void
+nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
+{
+        if (vma->node) {
+                if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) {
+                        spin_lock(&nvbo->bo.bdev->fence_lock);
+                        ttm_bo_wait(&nvbo->bo, false, false, false);
+                        spin_unlock(&nvbo->bo.bdev->fence_lock);
+                        nouveau_vm_unmap(vma);
+                }
+                nouveau_vm_put(vma);
+                list_del(&vma->head);
+        }
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_channel.c b/drivers/gpu/drm/nouveau/nouveau_channel.c
index a7583a8ddb01..b0d753f45bbd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_channel.c
+++ b/drivers/gpu/drm/nouveau/nouveau_channel.c
@@ -27,40 +27,63 @@
 #include "nouveau_drv.h"
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
+#include "nouveau_ramht.h"
 static int
-nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
+nouveau_channel_pushbuf_init(struct nouveau_channel *chan)
 {
+        u32 mem = nouveau_vram_pushbuf ? TTM_PL_FLAG_VRAM : TTM_PL_FLAG_TT;
        struct drm_device *dev = chan->dev;
        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_bo *pb = chan->pushbuf_bo;
+        int ret;
-        struct nouveau_gpuobj *pushbuf = NULL;
-        int ret = 0;
+        /* allocate buffer object */
+        ret = nouveau_bo_new(dev, 65536, 0, mem, 0, 0, &chan->pushbuf_bo);
+        if (ret)
+                goto out;
+        ret = nouveau_bo_pin(chan->pushbuf_bo, mem);
+        if (ret)
+                goto out;
+        ret = nouveau_bo_map(chan->pushbuf_bo);
+        if (ret)
+                goto out;
+        /* create DMA object covering the entire memtype where the push
+         * buffer resides, userspace can submit its own push buffers from
+         * anywhere within the same memtype.
+         */
+        chan->pushbuf_base = chan->pushbuf_bo->bo.offset;
        if (dev_priv->card_type >= NV_50) {
+                ret = nouveau_bo_vma_add(chan->pushbuf_bo, chan->vm,
+                                         &chan->pushbuf_vma);
+                if (ret)
+                        goto out;
                if (dev_priv->card_type < NV_C0) {
                        ret = nouveau_gpuobj_dma_new(chan,
                                                     NV_CLASS_DMA_IN_MEMORY, 0,
                                                     (1ULL << 40),
                                                     NV_MEM_ACCESS_RO,
                                                     NV_MEM_TARGET_VM,
-                                                     &pushbuf);
+                                                     &chan->pushbuf);
                }
-                chan->pushbuf_base = pb->bo.offset;
+                chan->pushbuf_base = chan->pushbuf_vma.offset;
        } else
-        if (pb->bo.mem.mem_type == TTM_PL_TT) {
+        if (chan->pushbuf_bo->bo.mem.mem_type == TTM_PL_TT) {
                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
                                             dev_priv->gart_info.aper_size,
                                             NV_MEM_ACCESS_RO,
-                                             NV_MEM_TARGET_GART, &pushbuf);
+                                             NV_MEM_TARGET_GART,
-                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
+                                             &chan->pushbuf);
        } else
        if (dev_priv->card_type != NV_04) {
                ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
                                             dev_priv->fb_available_size,
                                             NV_MEM_ACCESS_RO,
-                                             NV_MEM_TARGET_VRAM, &pushbuf);
+                                             NV_MEM_TARGET_VRAM,
-                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
+                                             &chan->pushbuf);
        } else {
                /* NV04 cmdbuf hack, from original ddx.. not sure of it's
                 * exact reason for existing :)  PCI access to cmdbuf in
@@ -70,47 +93,22 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
                                             pci_resource_start(dev->pdev, 1),
                                             dev_priv->fb_available_size,
                                             NV_MEM_ACCESS_RO,
-                                             NV_MEM_TARGET_PCI, &pushbuf);
+                                             NV_MEM_TARGET_PCI,
-                chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
+                                             &chan->pushbuf);
        }
-        nouveau_gpuobj_ref(pushbuf, &chan->pushbuf);
+out:
-        nouveau_gpuobj_ref(NULL, &pushbuf);
-        return ret;
-}
-static struct nouveau_bo *
-nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
-{
-        struct nouveau_bo *pushbuf = NULL;
-        int location, ret;
-        if (nouveau_vram_pushbuf)
-                location = TTM_PL_FLAG_VRAM;
-        else
-                location = TTM_PL_FLAG_TT;
-        ret = nouveau_bo_new(dev, NULL, 65536, 0, location, 0, 0x0000, &pushbuf);
-        if (ret) {
-                NV_ERROR(dev, "error allocating DMA push buffer: %d\n", ret);
-                return NULL;
-        }
-        ret = nouveau_bo_pin(pushbuf, location);
-        if (ret) {
-                NV_ERROR(dev, "error pinning DMA push buffer: %d\n", ret);
-                nouveau_bo_ref(NULL, &pushbuf);
-                return NULL;
-        }
-        ret = nouveau_bo_map(pushbuf);
        if (ret) {
-                nouveau_bo_unpin(pushbuf);
+                NV_ERROR(dev, "error initialising pushbuf: %d\n", ret);
-                nouveau_bo_ref(NULL, &pushbuf);
+                nouveau_bo_vma_del(chan->pushbuf_bo, &chan->pushbuf_vma);
-                return NULL;
+                nouveau_gpuobj_ref(NULL, &chan->pushbuf);
+                if (chan->pushbuf_bo) {
+                        nouveau_bo_unmap(chan->pushbuf_bo);
+                        nouveau_bo_ref(NULL, &chan->pushbuf_bo);
+                }
        }
-        return pushbuf;
+        return 0;
 }
 /* allocates and initializes a fifo for user space consumption */
@@ -121,6 +119,7 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
        struct nouveau_channel *chan;
        unsigned long flags;
        int ret;
@@ -160,19 +159,14 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
        INIT_LIST_HEAD(&chan->nvsw.flip);
        INIT_LIST_HEAD(&chan->fence.pending);
-        /* Allocate DMA push buffer */
+        /* setup channel's memory and vm */
-        chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev);
+        ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
-        if (!chan->pushbuf_bo) {
+        if (ret) {
-                ret = -ENOMEM;
+                NV_ERROR(dev, "gpuobj %d\n", ret);
-                NV_ERROR(dev, "pushbuf %d\n", ret);
                nouveau_channel_put(&chan);
                return ret;
        }
-        nouveau_dma_pre_init(chan);
-        chan->user_put = 0x40;
-        chan->user_get = 0x44;
        /* Allocate space for per-channel fixed notifier memory */
        ret = nouveau_notifier_init_channel(chan);
        if (ret) {
@@ -181,21 +175,17 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
                return ret;
        }
-        /* Setup channel's default objects */
+        /* Allocate DMA push buffer */
-        ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
+        ret = nouveau_channel_pushbuf_init(chan);
        if (ret) {
-                NV_ERROR(dev, "gpuobj %d\n", ret);
+                NV_ERROR(dev, "pushbuf %d\n", ret);
                nouveau_channel_put(&chan);
                return ret;
        }
-        /* Create a dma object for the push buffer */
+        nouveau_dma_pre_init(chan);
-        ret = nouveau_channel_pushbuf_ctxdma_init(chan);
+        chan->user_put = 0x40;
-        if (ret) {
+        chan->user_get = 0x44;
-                NV_ERROR(dev, "pbctxdma %d\n", ret);
-                nouveau_channel_put(&chan);
-                return ret;
-        }
        /* disable the fifo caches */
        pfifo->reassign(dev, false);
@@ -220,6 +210,11 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
        nouveau_debugfs_channel_init(chan);
        NV_DEBUG(dev, "channel %d initialised\n", chan->id);
+        if (fpriv) {
+                spin_lock(&fpriv->lock);
+                list_add(&chan->list, &fpriv->channels);
+                spin_unlock(&fpriv->lock);
+        }
        *chan_ret = chan;
        return 0;
 }
@@ -236,29 +231,23 @@ nouveau_channel_get_unlocked(struct nouveau_channel *ref)
 }
 struct nouveau_channel *
-nouveau_channel_get(struct drm_device *dev, struct drm_file *file_priv, int id)
+nouveau_channel_get(struct drm_file *file_priv, int id)
 {
-        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
        struct nouveau_channel *chan;
-        unsigned long flags;
-        if (unlikely(id < 0 || id >= NOUVEAU_MAX_CHANNEL_NR))
-                return ERR_PTR(-EINVAL);
-        spin_lock_irqsave(&dev_priv->channels.lock, flags);
-        chan = nouveau_channel_get_unlocked(dev_priv->channels.ptr[id]);
-        spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-        if (unlikely(!chan))
-                return ERR_PTR(-EINVAL);
-        if (unlikely(file_priv && chan->file_priv != file_priv)) {
+        spin_lock(&fpriv->lock);
-                nouveau_channel_put_unlocked(&chan);
+        list_for_each_entry(chan, &fpriv->channels, list) {
-                return ERR_PTR(-EINVAL);
+                if (chan->id == id) {
+                        chan = nouveau_channel_get_unlocked(chan);
+                        spin_unlock(&fpriv->lock);
+                        mutex_lock(&chan->mutex);
+                        return chan;
+                }
        }
+        spin_unlock(&fpriv->lock);
-        mutex_lock(&chan->mutex);
+        return ERR_PTR(-EINVAL);
-        return chan;
 }
 void
@@ -312,12 +301,14 @@ nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
        /* destroy any resources the channel owned */
        nouveau_gpuobj_ref(NULL, &chan->pushbuf);
        if (chan->pushbuf_bo) {
+                nouveau_bo_vma_del(chan->pushbuf_bo, &chan->pushbuf_vma);
                nouveau_bo_unmap(chan->pushbuf_bo);
                nouveau_bo_unpin(chan->pushbuf_bo);
                nouveau_bo_ref(NULL, &chan->pushbuf_bo);
        }
-        nouveau_gpuobj_channel_takedown(chan);
+        nouveau_ramht_ref(NULL, &chan->ramht, chan);
        nouveau_notifier_takedown_channel(chan);
+        nouveau_gpuobj_channel_takedown(chan);
        nouveau_channel_ref(NULL, pchan);
 }
@@ -383,10 +374,11 @@ nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv)
        NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
        for (i = 0; i < engine->fifo.channels; i++) {
-                chan = nouveau_channel_get(dev, file_priv, i);
+                chan = nouveau_channel_get(file_priv, i);
                if (IS_ERR(chan))
                        continue;
+                list_del(&chan->list);
                atomic_dec(&chan->users);
                nouveau_channel_put(&chan);
        }
@@ -459,10 +451,11 @@ nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
        struct drm_nouveau_channel_free *req = data;
        struct nouveau_channel *chan;
-        chan = nouveau_channel_get(dev, file_priv, req->channel);
+        chan = nouveau_channel_get(file_priv, req->channel);
        if (IS_ERR(chan))
                return PTR_ERR(chan);
+        list_del(&chan->list);
        atomic_dec(&chan->users);
        nouveau_channel_put(&chan);
        return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index 568caedd7216..00bc6eaad558 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -167,8 +167,13 @@ nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
              int delta, int length)
 {
        struct nouveau_bo *pb = chan->pushbuf_bo;
-        uint64_t offset = bo->bo.offset + delta;
+        struct nouveau_vma *vma;
        int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;
+        u64 offset;
+        vma = nouveau_bo_vma_find(bo, chan->vm);
+        BUG_ON(!vma);
+        offset = vma->offset + delta;
        BUG_ON(chan->dma.ib_free < 1);
        nouveau_bo_wr32(pb, ip++, lower_32_bits(offset));
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.c b/drivers/gpu/drm/nouveau/nouveau_drv.c
index 02c6f37d8bd7..8256370e5938 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.c
@@ -73,7 +73,7 @@ int nouveau_ignorelid = 0;
 module_param_named(ignorelid, nouveau_ignorelid, int, 0400);
 MODULE_PARM_DESC(noaccel, "Disable all acceleration");
-int nouveau_noaccel = 0;
+int nouveau_noaccel = -1;
 module_param_named(noaccel, nouveau_noaccel, int, 0400);
 MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
@@ -119,6 +119,10 @@ MODULE_PARM_DESC(msi, "Enable MSI (default: off)\n");
 int nouveau_msi;
 module_param_named(msi, nouveau_msi, int, 0400);
+MODULE_PARM_DESC(ctxfw, "Use external HUB/GPC ucode (fermi)\n");
+int nouveau_ctxfw;
+module_param_named(ctxfw, nouveau_ctxfw, int, 0400);
 int nouveau_fbpercrtc;
 #if 0
 module_param_named(fbpercrtc, nouveau_fbpercrtc, int, 0400);
@@ -354,7 +358,7 @@ nouveau_pci_resume(struct pci_dev *pdev)
        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-                u32 offset = nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT;
+                u32 offset = nv_crtc->cursor.nvbo->bo.offset;
                nv_crtc->cursor.set_offset(nv_crtc, offset);
                nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
@@ -389,7 +393,9 @@ static struct drm_driver driver = {
        .firstopen = nouveau_firstopen,
        .lastclose = nouveau_lastclose,
        .unload = nouveau_unload,
+        .open = nouveau_open,
        .preclose = nouveau_preclose,
+        .postclose = nouveau_postclose,
 #if defined(CONFIG_DRM_NOUVEAU_DEBUG)
        .debugfs_init = nouveau_debugfs_init,
        .debugfs_cleanup = nouveau_debugfs_takedown,
@@ -420,6 +426,8 @@ static struct drm_driver driver = {
        .gem_init_object = nouveau_gem_object_new,
        .gem_free_object = nouveau_gem_object_del,
+        .gem_open_object = nouveau_gem_object_open,
+        .gem_close_object = nouveau_gem_object_close,
        .name = DRIVER_NAME,
        .desc = DRIVER_DESC,
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index 9c56331941e2..72bfc143eb47 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -46,9 +46,17 @@
 #include "ttm/ttm_module.h"
 struct nouveau_fpriv {
-        struct ttm_object_file *tfile;
+        spinlock_t lock;
+        struct list_head channels;
+        struct nouveau_vm *vm;
 };
+static inline struct nouveau_fpriv *
+nouveau_fpriv(struct drm_file *file_priv)
+{
+        return file_priv ? file_priv->driver_priv : NULL;
+}
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 #include "nouveau_drm.h"
@@ -69,7 +77,7 @@ struct nouveau_mem {
        struct drm_device *dev;
        struct nouveau_vma bar_vma;
-        struct nouveau_vma tmp_vma;
+        struct nouveau_vma vma[2];
        u8  page_shift;
        struct drm_mm_node *tag;
@@ -107,7 +115,8 @@ struct nouveau_bo {
        struct nouveau_channel *channel;
-        struct nouveau_vma vma;
+        struct list_head vma_list;
+        unsigned page_shift;
        uint32_t tile_mode;
        uint32_t tile_flags;
@@ -176,9 +185,10 @@ struct nouveau_gpuobj {
        uint32_t flags;
        u32 size;
-        u32 pinst;
+        u32 pinst;      /* PRAMIN BAR offset */
-        u32 cinst;
+        u32 cinst;      /* Channel offset */
-        u64 vinst;
+        u64 vinst;      /* VRAM address */
+        u64 linst;      /* VM address */
        uint32_t engine;
        uint32_t class;
@@ -201,6 +211,7 @@ enum nouveau_channel_mutex_class {
 struct nouveau_channel {
        struct drm_device *dev;
+        struct list_head list;
        int id;
        /* references to the channel data structure */
@@ -228,15 +239,18 @@ struct nouveau_channel {
                uint32_t sequence;
                uint32_t sequence_ack;
                atomic_t last_sequence_irq;
+                struct nouveau_vma vma;
        } fence;
        /* DMA push buffer */
        struct nouveau_gpuobj *pushbuf;
        struct nouveau_bo     *pushbuf_bo;
+        struct nouveau_vma     pushbuf_vma;
        uint32_t               pushbuf_base;
        /* Notifier memory */
        struct nouveau_bo *notifier_bo;
+        struct nouveau_vma notifier_vma;
        struct drm_mm notifier_heap;
        /* PFIFO context */
@@ -278,6 +292,7 @@ struct nouveau_channel {
        uint32_t sw_subchannel[8];
+        struct nouveau_vma dispc_vma[2];
        struct {
                struct nouveau_gpuobj *vblsem;
                uint32_t vblsem_head;
@@ -314,7 +329,8 @@ struct nouveau_instmem_engine {
        int     (*suspend)(struct drm_device *dev);
        void    (*resume)(struct drm_device *dev);
-        int     (*get)(struct nouveau_gpuobj *, u32 size, u32 align);
+        int     (*get)(struct nouveau_gpuobj *, struct nouveau_channel *,
+                       u32 size, u32 align);
        void    (*put)(struct nouveau_gpuobj *);
        int     (*map)(struct nouveau_gpuobj *);
        void    (*unmap)(struct nouveau_gpuobj *);
@@ -445,9 +461,9 @@ struct nouveau_pm_level {
 struct nouveau_pm_temp_sensor_constants {
        u16 offset_constant;
        s16 offset_mult;
-        u16 offset_div;
+        s16 offset_div;
-        u16 slope_mult;
+        s16 slope_mult;
-        u16 slope_div;
+        s16 slope_div;
 };
 struct nouveau_pm_threshold_temp {
@@ -488,7 +504,10 @@ struct nouveau_pm_engine {
 };
 struct nouveau_vram_engine {
+        struct nouveau_mm *mm;
        int  (*init)(struct drm_device *);
+        void (*takedown)(struct drm_device *dev);
        int  (*get)(struct drm_device *, u64, u32 align, u32 size_nc,
                    u32 type, struct nouveau_mem **);
        void (*put)(struct drm_device *, struct nouveau_mem **);
@@ -608,6 +627,7 @@ enum nouveau_card_type {
 struct drm_nouveau_private {
        struct drm_device *dev;
+        bool noaccel;
        /* the card type, takes NV_* as values */
        enum nouveau_card_type card_type;
@@ -700,7 +720,6 @@ struct drm_nouveau_private {
        /* VRAM/fb configuration */
        uint64_t vram_size;
        uint64_t vram_sys_base;
-        u32 vram_rblock_size;
        uint64_t fb_phys;
        uint64_t fb_available_size;
@@ -784,12 +803,15 @@ extern int nouveau_override_conntype;
 extern char *nouveau_perflvl;
 extern int nouveau_perflvl_wr;
 extern int nouveau_msi;
+extern int nouveau_ctxfw;
 extern int nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state);
 extern int nouveau_pci_resume(struct pci_dev *pdev);
 /* nouveau_state.c */
+extern int  nouveau_open(struct drm_device *, struct drm_file *);
 extern void nouveau_preclose(struct drm_device *dev, struct drm_file *);
+extern void nouveau_postclose(struct drm_device *, struct drm_file *);
 extern int  nouveau_load(struct drm_device *, unsigned long flags);
 extern int  nouveau_firstopen(struct drm_device *);
 extern void nouveau_lastclose(struct drm_device *);
@@ -847,7 +869,7 @@ extern int  nouveau_channel_alloc(struct drm_device *dev,
 extern struct nouveau_channel *
 nouveau_channel_get_unlocked(struct nouveau_channel *);
 extern struct nouveau_channel *
-nouveau_channel_get(struct drm_device *, struct drm_file *, int id);
+nouveau_channel_get(struct drm_file *, int id);
 extern void nouveau_channel_put_unlocked(struct nouveau_channel **);
 extern void nouveau_channel_put(struct nouveau_channel **);
 extern void nouveau_channel_ref(struct nouveau_channel *chan,
@@ -1169,7 +1191,8 @@ extern int  nv04_instmem_init(struct drm_device *);
 extern void nv04_instmem_takedown(struct drm_device *);
 extern int  nv04_instmem_suspend(struct drm_device *);
 extern void nv04_instmem_resume(struct drm_device *);
-extern int  nv04_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
+extern int  nv04_instmem_get(struct nouveau_gpuobj *, struct nouveau_channel *,
+                             u32 size, u32 align);
 extern void nv04_instmem_put(struct nouveau_gpuobj *);
 extern int  nv04_instmem_map(struct nouveau_gpuobj *);
 extern void nv04_instmem_unmap(struct nouveau_gpuobj *);
@@ -1180,7 +1203,8 @@ extern int  nv50_instmem_init(struct drm_device *);
 extern void nv50_instmem_takedown(struct drm_device *);
 extern int  nv50_instmem_suspend(struct drm_device *);
 extern void nv50_instmem_resume(struct drm_device *);
-extern int  nv50_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
+extern int  nv50_instmem_get(struct nouveau_gpuobj *, struct nouveau_channel *,
+                             u32 size, u32 align);
 extern void nv50_instmem_put(struct nouveau_gpuobj *);
 extern int  nv50_instmem_map(struct nouveau_gpuobj *);
 extern void nv50_instmem_unmap(struct nouveau_gpuobj *);
@@ -1247,10 +1271,9 @@ extern int nv04_crtc_create(struct drm_device *, int index);
 /* nouveau_bo.c */
 extern struct ttm_bo_driver nouveau_bo_driver;
-extern int nouveau_bo_new(struct drm_device *, struct nouveau_channel *,
+extern int nouveau_bo_new(struct drm_device *, int size, int align,
-                          int size, int align, uint32_t flags,
+                          uint32_t flags, uint32_t tile_mode,
-                          uint32_t tile_mode, uint32_t tile_flags,
+                          uint32_t tile_flags, struct nouveau_bo **);
-                          struct nouveau_bo **);
 extern int nouveau_bo_pin(struct nouveau_bo *, uint32_t flags);
 extern int nouveau_bo_unpin(struct nouveau_bo *);
 extern int nouveau_bo_map(struct nouveau_bo *);
@@ -1265,6 +1288,12 @@ extern void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *);
 extern int nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
                               bool no_wait_reserve, bool no_wait_gpu);
+extern struct nouveau_vma *
+nouveau_bo_vma_find(struct nouveau_bo *, struct nouveau_vm *);
+extern int  nouveau_bo_vma_add(struct nouveau_bo *, struct nouveau_vm *,
+                               struct nouveau_vma *);
+extern void nouveau_bo_vma_del(struct nouveau_bo *, struct nouveau_vma *);
 /* nouveau_fence.c */
 struct nouveau_fence;
 extern int nouveau_fence_init(struct drm_device *);
@@ -1310,12 +1339,14 @@ static inline struct nouveau_fence *nouveau_fence_ref(struct nouveau_fence *obj)
 }
 /* nouveau_gem.c */
-extern int nouveau_gem_new(struct drm_device *, struct nouveau_channel *,
+extern int nouveau_gem_new(struct drm_device *, int size, int align,
-                           int size, int align, uint32_t domain,
+                           uint32_t domain, uint32_t tile_mode,
-                           uint32_t tile_mode, uint32_t tile_flags,
+                           uint32_t tile_flags, struct nouveau_bo **);
-                           struct nouveau_bo **);
 extern int nouveau_gem_object_new(struct drm_gem_object *);
 extern void nouveau_gem_object_del(struct drm_gem_object *);
+extern int nouveau_gem_object_open(struct drm_gem_object *, struct drm_file *);
+extern void nouveau_gem_object_close(struct drm_gem_object *,
+                                     struct drm_file *);
 extern int nouveau_gem_ioctl_new(struct drm_device *, void *,
                                 struct drm_file *);
 extern int nouveau_gem_ioctl_pushbuf(struct drm_device *, void *,
diff --git a/drivers/gpu/drm/nouveau/nouveau_fb.h b/drivers/gpu/drm/nouveau/nouveau_fb.h
index a3a88ad00f86..95c843e684bb 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fb.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fb.h
@@ -30,6 +30,7 @@
 struct nouveau_framebuffer {
        struct drm_framebuffer base;
        struct nouveau_bo *nvbo;
+        struct nouveau_vma vma;
        u32 r_dma;
        u32 r_format;
        u32 r_pitch;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 39aee6d4daf8..14a8627efe4d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -279,6 +279,7 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
        struct fb_info *info;
        struct drm_framebuffer *fb;
        struct nouveau_framebuffer *nouveau_fb;
+        struct nouveau_channel *chan;
        struct nouveau_bo *nvbo;
        struct drm_mode_fb_cmd mode_cmd;
        struct pci_dev *pdev = dev->pdev;
@@ -296,8 +297,8 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
        size = mode_cmd.pitch * mode_cmd.height;
        size = roundup(size, PAGE_SIZE);
-        ret = nouveau_gem_new(dev, dev_priv->channel, size, 0,
+        ret = nouveau_gem_new(dev, size, 0, NOUVEAU_GEM_DOMAIN_VRAM,
-                              NOUVEAU_GEM_DOMAIN_VRAM, 0, 0x0000, &nvbo);
+                              0, 0x0000, &nvbo);
        if (ret) {
                NV_ERROR(dev, "failed to allocate framebuffer\n");
                goto out;
@@ -318,6 +319,15 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
                goto out;
        }
+        chan = nouveau_nofbaccel ? NULL : dev_priv->channel;
+        if (chan && dev_priv->card_type >= NV_50) {
+                ret = nouveau_bo_vma_add(nvbo, chan->vm, &nfbdev->nouveau_fb.vma);
+                if (ret) {
+                        NV_ERROR(dev, "failed to map fb into chan: %d\n", ret);
+                        chan = NULL;
+                }
+        }
        mutex_lock(&dev->struct_mutex);
        info = framebuffer_alloc(0, device);
@@ -448,6 +458,7 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
        if (nouveau_fb->nvbo) {
                nouveau_bo_unmap(nouveau_fb->nvbo);
+                nouveau_bo_vma_del(nouveau_fb->nvbo, &nouveau_fb->vma);
                drm_gem_object_unreference_unlocked(nouveau_fb->nvbo->gem);
                nouveau_fb->nvbo = NULL;
        }
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 7347075ca5b8..8d02d875376d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -336,6 +336,7 @@ semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 {
        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
        struct nouveau_fence *fence = NULL;
+        u64 offset = chan->fence.vma.offset + sema->mem->start;
        int ret;
        if (dev_priv->chipset < 0x84) {
@@ -345,13 +346,10 @@ semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
                BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 3);
                OUT_RING  (chan, NvSema);
-                OUT_RING  (chan, sema->mem->start);
+                OUT_RING  (chan, offset);
                OUT_RING  (chan, 1);
        } else
        if (dev_priv->chipset < 0xc0) {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
                ret = RING_SPACE(chan, 7);
                if (ret)
                        return ret;
@@ -364,9 +362,6 @@ semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
                OUT_RING  (chan, 1);
                OUT_RING  (chan, 1); /* ACQUIRE_EQ */
        } else {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
                ret = RING_SPACE(chan, 5);
                if (ret)
                        return ret;
@@ -394,6 +389,7 @@ semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
 {
        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
        struct nouveau_fence *fence = NULL;
+        u64 offset = chan->fence.vma.offset + sema->mem->start;
        int ret;
        if (dev_priv->chipset < 0x84) {
@@ -403,14 +399,11 @@ semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
                BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 2);
                OUT_RING  (chan, NvSema);
-                OUT_RING  (chan, sema->mem->start);
+                OUT_RING  (chan, offset);
                BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_RELEASE, 1);
                OUT_RING  (chan, 1);
        } else
        if (dev_priv->chipset < 0xc0) {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
                ret = RING_SPACE(chan, 7);
                if (ret)
                        return ret;
@@ -423,9 +416,6 @@ semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
                OUT_RING  (chan, 1);
                OUT_RING  (chan, 2); /* RELEASE */
        } else {
-                struct nouveau_vma *vma = &dev_priv->fence.bo->vma;
-                u64 offset = vma->offset + sema->mem->start;
                ret = RING_SPACE(chan, 5);
                if (ret)
                        return ret;
@@ -540,6 +530,12 @@ nouveau_fence_channel_init(struct nouveau_channel *chan)
                nouveau_gpuobj_ref(NULL, &obj);
                if (ret)
                        return ret;
+        } else {
+                /* map fence bo into channel's vm */
+                ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
+                                         &chan->fence.vma);
+                if (ret)
+                        return ret;
        }
        INIT_LIST_HEAD(&chan->fence.pending);
@@ -551,10 +547,10 @@ nouveau_fence_channel_init(struct nouveau_channel *chan)
 void
 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 {
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
        struct nouveau_fence *tmp, *fence;
        spin_lock(&chan->fence.lock);
        list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
                fence->signalled = true;
                list_del(&fence->entry);
@@ -564,8 +560,9 @@ nouveau_fence_channel_fini(struct nouveau_channel *chan)
                kref_put(&fence->refcount, nouveau_fence_del);
        }
        spin_unlock(&chan->fence.lock);
+        nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
 }
 int
@@ -577,7 +574,7 @@ nouveau_fence_init(struct drm_device *dev)
        /* Create a shared VRAM heap for cross-channel sync. */
        if (USE_SEMA(dev)) {
-                ret = nouveau_bo_new(dev, NULL, size, 0, TTM_PL_FLAG_VRAM,
+                ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
                                     0, 0, &dev_priv->fence.bo);
                if (ret)
                        return ret;
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index b52e46018245..5f0bc57fdaab 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -60,9 +60,71 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
 }
 int
-nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
+nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
-                int size, int align, uint32_t domain, uint32_t tile_mode,
+{
-                uint32_t tile_flags, struct nouveau_bo **pnvbo)
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+        struct nouveau_bo *nvbo = nouveau_gem_object(gem);
+        struct nouveau_vma *vma;
+        int ret;
+        if (!fpriv->vm)
+                return 0;
+        ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
+        if (ret)
+                return ret;
+        vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+        if (!vma) {
+                vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+                if (!vma) {
+                        ret = -ENOMEM;
+                        goto out;
+                }
+                ret = nouveau_bo_vma_add(nvbo, fpriv->vm, vma);
+                if (ret) {
+                        kfree(vma);
+                        goto out;
+                }
+        } else {
+                vma->refcount++;
+        }
+out:
+        ttm_bo_unreserve(&nvbo->bo);
+        return ret;
+}
+void
+nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
+{
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+        struct nouveau_bo *nvbo = nouveau_gem_object(gem);
+        struct nouveau_vma *vma;
+        int ret;
+        if (!fpriv->vm)
+                return;
+        ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
+        if (ret)
+                return;
+        vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+        if (vma) {
+                if (--vma->refcount == 0) {
+                        nouveau_bo_vma_del(nvbo, vma);
+                        kfree(vma);
+                }
+        }
+        ttm_bo_unreserve(&nvbo->bo);
+}
+int
+nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
+                uint32_t tile_mode, uint32_t tile_flags,
+                struct nouveau_bo **pnvbo)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_bo *nvbo;
@@ -76,7 +138,7 @@ nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
        if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU)
                flags |= TTM_PL_FLAG_SYSTEM;
-        ret = nouveau_bo_new(dev, chan, size, align, flags, tile_mode,
+        ret = nouveau_bo_new(dev, size, align, flags, tile_mode,
                             tile_flags, pnvbo);
        if (ret)
                return ret;
@@ -103,17 +165,28 @@ nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
 }
 static int
-nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep)
+nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
+                 struct drm_nouveau_gem_info *rep)
 {
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
        struct nouveau_bo *nvbo = nouveau_gem_object(gem);
+        struct nouveau_vma *vma;
        if (nvbo->bo.mem.mem_type == TTM_PL_TT)
                rep->domain = NOUVEAU_GEM_DOMAIN_GART;
        else
                rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
-        rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
        rep->offset = nvbo->bo.offset;
+        if (fpriv->vm) {
+                vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+                if (!vma)
+                        return -EINVAL;
+                rep->offset = vma->offset;
+        }
+        rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
        rep->map_handle = nvbo->bo.addr_space_offset;
        rep->tile_mode = nvbo->tile_mode;
        rep->tile_flags = nvbo->tile_flags;
@@ -127,7 +200,6 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct drm_nouveau_gem_new *req = data;
        struct nouveau_bo *nvbo = NULL;
-        struct nouveau_channel *chan = NULL;
        int ret = 0;
        if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL))
@@ -138,28 +210,21 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
                return -EINVAL;
        }
-        if (req->channel_hint) {
+        ret = nouveau_gem_new(dev, req->info.size, req->align,
-                chan = nouveau_channel_get(dev, file_priv, req->channel_hint);
-                if (IS_ERR(chan))
-                        return PTR_ERR(chan);
-        }
-        ret = nouveau_gem_new(dev, chan, req->info.size, req->align,
                              req->info.domain, req->info.tile_mode,
                              req->info.tile_flags, &nvbo);
-        if (chan)
-                nouveau_channel_put(&chan);
        if (ret)
                return ret;
-        ret = nouveau_gem_info(nvbo->gem, &req->info);
-        if (ret)
-                goto out;
        ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle);
+        if (ret == 0) {
+                ret = nouveau_gem_info(file_priv, nvbo->gem, &req->info);
+                if (ret)
+                        drm_gem_handle_delete(file_priv, req->info.handle);
+        }
        /* drop reference from allocate - handle holds it now */
        drm_gem_object_unreference_unlocked(nvbo->gem);
-out:
        return ret;
 }
@@ -318,6 +383,7 @@ static int
 validate_list(struct nouveau_channel *chan, struct list_head *list,
              struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr)
 {
+        struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
        struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
                                (void __force __user *)(uintptr_t)user_pbbo_ptr;
        struct drm_device *dev = chan->dev;
@@ -356,24 +422,26 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
                        return ret;
                }
-                if (nvbo->bo.offset == b->presumed.offset &&
+                if (dev_priv->card_type < NV_50) {
-                    ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
+                        if (nvbo->bo.offset == b->presumed.offset &&
-                      b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
+                            ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
-                     (nvbo->bo.mem.mem_type == TTM_PL_TT &&
+                              b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
-                      b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
+                             (nvbo->bo.mem.mem_type == TTM_PL_TT &&
-                        continue;
+                              b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
+                                continue;
-                if (nvbo->bo.mem.mem_type == TTM_PL_TT)
+                        if (nvbo->bo.mem.mem_type == TTM_PL_TT)
-                        b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
+                                b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
-                else
+                        else
-                        b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
+                                b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
-                b->presumed.offset = nvbo->bo.offset;
+                        b->presumed.offset = nvbo->bo.offset;
-                b->presumed.valid = 0;
+                        b->presumed.valid = 0;
-                relocs++;
+                        relocs++;
-                if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed,
+                        if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed,
-                                     &b->presumed, sizeof(b->presumed)))
+                                             &b->presumed, sizeof(b->presumed)))
-                        return -EFAULT;
+                                return -EFAULT;
+                }
        }
        return relocs;
@@ -548,7 +616,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
        struct nouveau_fence *fence = NULL;
        int i, j, ret = 0, do_reloc = 0;
-        chan = nouveau_channel_get(dev, file_priv, req->channel);
+        chan = nouveau_channel_get(file_priv, req->channel);
        if (IS_ERR(chan))
                return PTR_ERR(chan);
@@ -782,7 +850,7 @@ nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
        if (!gem)
                return -ENOENT;
-        ret = nouveau_gem_info(gem, req);
+        ret = nouveau_gem_info(file_priv, gem, req);
        drm_gem_object_unreference_unlocked(gem);
        return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 5ee14d216ce8..f9ae2fc3d6f1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -397,7 +397,7 @@ nouveau_mem_vram_init(struct drm_device *dev)
                if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(40)))
                        dma_bits = 40;
        } else
-        if (0 && drm_pci_device_is_pcie(dev) &&
+        if (0 && pci_is_pcie(dev->pdev) &&
            dev_priv->chipset  > 0x40 &&
            dev_priv->chipset != 0x45) {
                if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(39)))
@@ -423,38 +423,6 @@ nouveau_mem_vram_init(struct drm_device *dev)
                return ret;
        }
-        /* reserve space at end of VRAM for PRAMIN */
-        if (dev_priv->card_type >= NV_50) {
-                dev_priv->ramin_rsvd_vram = 1 * 1024 * 1024;
-        } else
-        if (dev_priv->card_type >= NV_40) {
-                u32 vs = hweight8((nv_rd32(dev, 0x001540) & 0x0000ff00) >> 8);
-                u32 rsvd;
-                /* estimate grctx size, the magics come from nv40_grctx.c */
-                if      (dev_priv->chipset == 0x40) rsvd = 0x6aa0 * vs;
-                else if (dev_priv->chipset  < 0x43) rsvd = 0x4f00 * vs;
-                else if (nv44_graph_class(dev))     rsvd = 0x4980 * vs;
-                else                                rsvd = 0x4a40 * vs;
-                rsvd += 16 * 1024;
-                rsvd *= dev_priv->engine.fifo.channels;
-                /* pciegart table */
-                if (drm_pci_device_is_pcie(dev))
-                        rsvd += 512 * 1024;
-                /* object storage */
-                rsvd += 512 * 1024;
-                dev_priv->ramin_rsvd_vram = round_up(rsvd, 4096);
-        } else {
-                dev_priv->ramin_rsvd_vram = 512 * 1024;
-        }
-        ret = dev_priv->engine.vram.init(dev);
-        if (ret)
-                return ret;
        NV_INFO(dev, "Detected %dMiB VRAM\n", (int)(dev_priv->vram_size >> 20));
        if (dev_priv->vram_sys_base) {
                NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
@@ -479,7 +447,7 @@ nouveau_mem_vram_init(struct drm_device *dev)
        }
        if (dev_priv->card_type < NV_50) {
-                ret = nouveau_bo_new(dev, NULL, 256*1024, 0, TTM_PL_FLAG_VRAM,
+                ret = nouveau_bo_new(dev, 256*1024, 0, TTM_PL_FLAG_VRAM,
                                     0, 0, &dev_priv->vga_ram);
                if (ret == 0)
                        ret = nouveau_bo_pin(dev_priv->vga_ram,
@@ -729,37 +697,31 @@ nouveau_mem_timing_fini(struct drm_device *dev)
 }
 static int
-nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long p_size)
+nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
 {
-        struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
+        /* nothing to do */
-        struct nouveau_mm *mm;
-        u64 size, block, rsvd;
-        int ret;
-        rsvd  = (256 * 1024); /* vga memory */
-        size  = (p_size << PAGE_SHIFT) - rsvd;
-        block = dev_priv->vram_rblock_size;
-        ret = nouveau_mm_init(&mm, rsvd >> 12, size >> 12, block >> 12);
-        if (ret)
-                return ret;
-        man->priv = mm;
        return 0;
 }
 static int
 nouveau_vram_manager_fini(struct ttm_mem_type_manager *man)
 {
-        struct nouveau_mm *mm = man->priv;
+        /* nothing to do */
-        int ret;
+        return 0;
+}
-        ret = nouveau_mm_fini(&mm);
+static inline void
-        if (ret)
+nouveau_mem_node_cleanup(struct nouveau_mem *node)
-                return ret;
+{
+        if (node->vma[0].node) {
+                nouveau_vm_unmap(&node->vma[0]);
+                nouveau_vm_put(&node->vma[0]);
+        }
-        man->priv = NULL;
+        if (node->vma[1].node) {
-        return 0;
+                nouveau_vm_unmap(&node->vma[1]);
+                nouveau_vm_put(&node->vma[1]);
+        }
 }
 static void
@@ -768,14 +730,9 @@ nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
 {
        struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-        struct nouveau_mem *node = mem->mm_node;
        struct drm_device *dev = dev_priv->dev;
-        if (node->tmp_vma.node) {
+        nouveau_mem_node_cleanup(mem->mm_node);
-                nouveau_vm_unmap(&node->tmp_vma);
-                nouveau_vm_put(&node->tmp_vma);
-        }
        vram->put(dev, (struct nouveau_mem **)&mem->mm_node);
 }
@@ -794,7 +751,7 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
        int ret;
        if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)
-                size_nc = 1 << nvbo->vma.node->type;
+                size_nc = 1 << nvbo->page_shift;
        ret = vram->get(dev, mem->num_pages << PAGE_SHIFT,
                        mem->page_alignment << PAGE_SHIFT, size_nc,
@@ -804,9 +761,7 @@ nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
                return (ret == -ENOSPC) ? 0 : ret;
        }
-        node->page_shift = 12;
+        node->page_shift = nvbo->page_shift;
-        if (nvbo->vma.node)
-                node->page_shift = nvbo->vma.node->type;
        mem->mm_node = node;
        mem->start   = node->offset >> PAGE_SHIFT;
@@ -862,15 +817,9 @@ static void
 nouveau_gart_manager_del(struct ttm_mem_type_manager *man,
                         struct ttm_mem_reg *mem)
 {
-        struct nouveau_mem *node = mem->mm_node;
+        nouveau_mem_node_cleanup(mem->mm_node);
+        kfree(mem->mm_node);
-        if (node->tmp_vma.node) {
-                nouveau_vm_unmap(&node->tmp_vma);
-                nouveau_vm_put(&node->tmp_vma);
-        }
        mem->mm_node = NULL;
-        kfree(node);
 }
 static int
@@ -880,11 +829,7 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
                         struct ttm_mem_reg *mem)
 {
        struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-        struct nouveau_bo *nvbo = nouveau_bo(bo);
-        struct nouveau_vma *vma = &nvbo->vma;
-        struct nouveau_vm *vm = vma->vm;
        struct nouveau_mem *node;
-        int ret;
        if (unlikely((mem->num_pages << PAGE_SHIFT) >=
                     dev_priv->gart_info.aper_size))
@@ -893,24 +838,8 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
        node = kzalloc(sizeof(*node), GFP_KERNEL);
        if (!node)
                return -ENOMEM;
+        node->page_shift = 12;
-        /* This node must be for evicting large-paged VRAM
-         * to system memory.  Due to a nv50 limitation of
-         * not being able to mix large/small pages within
-         * the same PDE, we need to create a temporary
-         * small-paged VMA for the eviction.
-         */
-        if (vma->node->type != vm->spg_shift) {
-                ret = nouveau_vm_get(vm, (u64)vma->node->length << 12,
-                                     vm->spg_shift, NV_MEM_ACCESS_RW,
-                                     &node->tmp_vma);
-                if (ret) {
-                        kfree(node);
-                        return ret;
-                }
-        }
-        node->page_shift = nvbo->vma.node->type;
        mem->mm_node = node;
        mem->start   = 0;
        return 0;
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.c b/drivers/gpu/drm/nouveau/nouveau_mm.c
index 7609756b6faf..1640dec3b823 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.c
@@ -158,11 +158,18 @@ int
 nouveau_mm_fini(struct nouveau_mm **prmm)
 {
        struct nouveau_mm *rmm = *prmm;
-        struct nouveau_mm_node *heap =
+        struct nouveau_mm_node *node, *heap =
                list_first_entry(&rmm->nodes, struct nouveau_mm_node, nl_entry);
-        if (!list_is_singular(&rmm->nodes))
+        if (!list_is_singular(&rmm->nodes)) {
+                printk(KERN_ERR "nouveau_mm not empty at destroy time!\n");
+                list_for_each_entry(node, &rmm->nodes, nl_entry) {
+                        printk(KERN_ERR "0x%02x: 0x%08x 0x%08x\n",
+                               node->type, node->offset, node->length);
+                }
+                WARN_ON(1);
                return -EBUSY;
+        }
        kfree(heap);
        kfree(rmm);
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.h b/drivers/gpu/drm/nouveau/nouveau_mm.h
index 1f7483aae9a4..b9c016d21553 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.h
@@ -52,6 +52,7 @@ int  nouveau_mm_get(struct nouveau_mm *, int type, u32 size, u32 size_nc,
 void nouveau_mm_put(struct nouveau_mm *, struct nouveau_mm_node *);
 int  nv50_vram_init(struct drm_device *);
+void nv50_vram_fini(struct drm_device *);
 int  nv50_vram_new(struct drm_device *, u64 size, u32 align, u32 size_nc,
                    u32 memtype, struct nouveau_mem **);
 void nv50_vram_del(struct drm_device *, struct nouveau_mem **);
diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c
index 5b39718ae1f8..6abdbe6530a7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_notifier.c
+++ b/drivers/gpu/drm/nouveau/nouveau_notifier.c
@@ -34,6 +34,7 @@ int
 nouveau_notifier_init_channel(struct nouveau_channel *chan)
 {
        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_bo *ntfy = NULL;
        uint32_t flags, ttmpl;
        int ret;
@@ -46,7 +47,7 @@ nouveau_notifier_init_channel(struct nouveau_channel *chan)
                ttmpl = TTM_PL_FLAG_TT;
        }
-        ret = nouveau_gem_new(dev, NULL, PAGE_SIZE, 0, flags, 0, 0, &ntfy);
+        ret = nouveau_gem_new(dev, PAGE_SIZE, 0, flags, 0, 0, &ntfy);
        if (ret)
                return ret;
@@ -58,14 +59,22 @@ nouveau_notifier_init_channel(struct nouveau_channel *chan)
        if (ret)
                goto out_err;
+        if (dev_priv->card_type >= NV_50) {
+                ret = nouveau_bo_vma_add(ntfy, chan->vm, &chan->notifier_vma);
+                if (ret)
+                        goto out_err;
+        }
        ret = drm_mm_init(&chan->notifier_heap, 0, ntfy->bo.mem.size);
        if (ret)
                goto out_err;
        chan->notifier_bo = ntfy;
 out_err:
-        if (ret)
+        if (ret) {
+                nouveau_bo_vma_del(ntfy, &chan->notifier_vma);
                drm_gem_object_unreference_unlocked(ntfy->gem);
+        }
        return ret;
 }
@@ -78,6 +87,7 @@ nouveau_notifier_takedown_channel(struct nouveau_channel *chan)
        if (!chan->notifier_bo)
                return;
+        nouveau_bo_vma_del(chan->notifier_bo, &chan->notifier_vma);
        nouveau_bo_unmap(chan->notifier_bo);
        mutex_lock(&dev->struct_mutex);
        nouveau_bo_unpin(chan->notifier_bo);
@@ -122,10 +132,10 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
                        target = NV_MEM_TARGET_VRAM;
                else
                        target = NV_MEM_TARGET_GART;
-                offset  = chan->notifier_bo->bo.mem.start << PAGE_SHIFT;
+                offset  = chan->notifier_bo->bo.offset;
        } else {
                target = NV_MEM_TARGET_VM;
-                offset = chan->notifier_bo->vma.offset;
+                offset = chan->notifier_vma.offset;
        }
        offset += mem->start;
@@ -183,7 +193,7 @@ nouveau_ioctl_notifier_alloc(struct drm_device *dev, void *data,
        if (unlikely(dev_priv->card_type >= NV_C0))
                return -EINVAL;
-        chan = nouveau_channel_get(dev, file_priv, na->channel);
+        chan = nouveau_channel_get(file_priv, na->channel);
        if (IS_ERR(chan))
                return PTR_ERR(chan);
diff --git a/drivers/gpu/drm/nouveau/nouveau_object.c b/drivers/gpu/drm/nouveau/nouveau_object.c
index 8f97016f5b26..159b7c437d3f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_object.c
+++ b/drivers/gpu/drm/nouveau/nouveau_object.c
@@ -125,7 +125,7 @@ nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
        int ret = -EINVAL;
        spin_lock_irqsave(&dev_priv->channels.lock, flags);
-        if (chid > 0 && chid < dev_priv->engine.fifo.channels)
+        if (chid >= 0 && chid < dev_priv->engine.fifo.channels)
                chan = dev_priv->channels.ptr[chid];
        if (chan)
                ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
@@ -191,7 +191,7 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
        list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
        spin_unlock(&dev_priv->ramin_lock);
-        if (chan) {
+        if (!(flags & NVOBJ_FLAG_VM) && chan) {
                ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
                if (ramin)
                        ramin = drm_mm_get_block(ramin, size, align);
@@ -208,7 +208,7 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
                gpuobj->vinst = ramin->start + chan->ramin->vinst;
                gpuobj->node  = ramin;
        } else {
-                ret = instmem->get(gpuobj, size, align);
+                ret = instmem->get(gpuobj, chan, size, align);
                if (ret) {
                        nouveau_gpuobj_ref(NULL, &gpuobj);
                        return ret;
@@ -690,35 +690,64 @@ nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
        return 0;
 }
+static int
+nvc0_gpuobj_channel_init(struct nouveau_channel *chan, struct nouveau_vm *vm)
+{
+        struct drm_device *dev = chan->dev;
+        struct nouveau_gpuobj *pgd = NULL;
+        struct nouveau_vm_pgd *vpgd;
+        int ret, i;
+        ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0, &chan->ramin);
+        if (ret)
+                return ret;
+        /* create page directory for this vm if none currently exists,
+         * will be destroyed automagically when last reference to the
+         * vm is removed
+         */
+        if (list_empty(&vm->pgd_list)) {
+                ret = nouveau_gpuobj_new(dev, NULL, 65536, 0x1000, 0, &pgd);
+                if (ret)
+                        return ret;
+        }
+        nouveau_vm_ref(vm, &chan->vm, pgd);
+        nouveau_gpuobj_ref(NULL, &pgd);
+        /* point channel at vm's page directory */
+        vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
+        nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
+        nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
+        nv_wo32(chan->ramin, 0x0208, 0xffffffff);
+        nv_wo32(chan->ramin, 0x020c, 0x000000ff);
+        /* map display semaphore buffers into channel's vm */
+        for (i = 0; i < 2; i++) {
+                struct nv50_display_crtc *dispc = &nv50_display(dev)->crtc[i];
+                ret = nouveau_bo_vma_add(dispc->sem.bo, chan->vm,
+                                         &chan->dispc_vma[i]);
+                if (ret)
+                        return ret;
+        }
+        return 0;
+}
 int
 nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                            uint32_t vram_h, uint32_t tt_h)
 {
        struct drm_device *dev = chan->dev;
        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(chan->file_priv);
+        struct nouveau_vm *vm = fpriv ? fpriv->vm : dev_priv->chan_vm;
        struct nouveau_gpuobj *vram = NULL, *tt = NULL;
        int ret, i;
        NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
+        if (dev_priv->card_type == NV_C0)
-        if (dev_priv->card_type == NV_C0) {
+                return nvc0_gpuobj_channel_init(chan, vm);
-                struct nouveau_vm *vm = dev_priv->chan_vm;
-                struct nouveau_vm_pgd *vpgd;
-                ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0,
-                                         &chan->ramin);
-                if (ret)
-                        return ret;
-                nouveau_vm_ref(vm, &chan->vm, NULL);
-                vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
-                nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
-                nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
-                nv_wo32(chan->ramin, 0x0208, 0xffffffff);
-                nv_wo32(chan->ramin, 0x020c, 0x000000ff);
-                return 0;
-        }
        /* Allocate a chunk of memory for per-channel object storage */
        ret = nouveau_gpuobj_channel_init_pramin(chan);
@@ -731,7 +760,7 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
         *  - Allocate per-channel page-directory
         *  - Link with shared channel VM
         */
-        if (dev_priv->chan_vm) {
+        if (vm) {
                u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
                u64 vm_vinst = chan->ramin->vinst + pgd_offs;
                u32 vm_pinst = chan->ramin->pinst;
@@ -744,7 +773,7 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                if (ret)
                        return ret;
-                nouveau_vm_ref(dev_priv->chan_vm, &chan->vm, chan->vm_pd);
+                nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
        }
        /* RAMHT */
@@ -768,7 +797,7 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
                        struct nouveau_gpuobj *sem = NULL;
                        struct nv50_display_crtc *dispc =
                                &nv50_display(dev)->crtc[i];
-                        u64 offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
+                        u64 offset = dispc->sem.bo->bo.offset;
                        ret = nouveau_gpuobj_dma_new(chan, 0x3d, offset, 0xfff,
                                                     NV_MEM_ACCESS_RW,
@@ -841,13 +870,22 @@ void
 nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
 {
        struct drm_device *dev = chan->dev;
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int i;
        NV_DEBUG(dev, "ch%d\n", chan->id);
-        nouveau_ramht_ref(NULL, &chan->ramht, chan);
+        if (dev_priv->card_type >= NV_50) {
+                struct nv50_display *disp = nv50_display(dev);
+                for (i = 0; i < 2; i++) {
+                        struct nv50_display_crtc *dispc = &disp->crtc[i];
+                        nouveau_bo_vma_del(dispc->sem.bo, &chan->dispc_vma[i]);
+                }
-        nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
+                nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
-        nouveau_gpuobj_ref(NULL, &chan->vm_pd);
+                nouveau_gpuobj_ref(NULL, &chan->vm_pd);
+        }
        if (drm_mm_initialized(&chan->ramin_heap))
                drm_mm_takedown(&chan->ramin_heap);
@@ -909,7 +947,7 @@ int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
        if (init->handle == ~0)
                return -EINVAL;
-        chan = nouveau_channel_get(dev, file_priv, init->channel);
+        chan = nouveau_channel_get(file_priv, init->channel);
        if (IS_ERR(chan))
                return PTR_ERR(chan);
@@ -936,7 +974,7 @@ int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
        struct nouveau_channel *chan;
        int ret;
-        chan = nouveau_channel_get(dev, file_priv, objfree->channel);
+        chan = nouveau_channel_get(file_priv, objfree->channel);
        if (IS_ERR(chan))
                return PTR_ERR(chan);
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index 82fad914e648..c444cadbf849 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -429,7 +429,7 @@ nouveau_sgdma_init(struct drm_device *dev)
        u32 aper_size, align;
        int ret;
-        if (dev_priv->card_type >= NV_40 && drm_pci_device_is_pcie(dev))
+        if (dev_priv->card_type >= NV_40 && pci_is_pcie(dev->pdev))
                aper_size = 512 * 1024 * 1024;
        else
                aper_size = 64 * 1024 * 1024;
@@ -458,7 +458,7 @@ nouveau_sgdma_init(struct drm_device *dev)
                dev_priv->gart_info.type = NOUVEAU_GART_HW;
                dev_priv->gart_info.func = &nv50_sgdma_backend;
        } else
-        if (0 && drm_pci_device_is_pcie(dev) &&
+        if (0 && pci_is_pcie(dev->pdev) &&
            dev_priv->chipset > 0x40 && dev_priv->chipset != 0x45) {
                if (nv44_graph_class(dev)) {
                        dev_priv->gart_info.func = &nv44_sgdma_backend;
diff --git a/drivers/gpu/drm/nouveau/nouveau_state.c b/drivers/gpu/drm/nouveau/nouveau_state.c
index 731acea865b5..fa1e63210040 100644
--- a/drivers/gpu/drm/nouveau/nouveau_state.c
+++ b/drivers/gpu/drm/nouveau/nouveau_state.c
@@ -91,6 +91,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->pm.clock_pre            = nv04_pm_clock_pre;
                engine->pm.clock_set            = nv04_pm_clock_set;
                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                break;
        case 0x10:
@@ -139,6 +140,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->pm.clock_pre            = nv04_pm_clock_pre;
                engine->pm.clock_set            = nv04_pm_clock_set;
                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                break;
        case 0x20:
@@ -187,6 +189,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->pm.clock_pre            = nv04_pm_clock_pre;
                engine->pm.clock_set            = nv04_pm_clock_set;
                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                break;
        case 0x30:
@@ -237,6 +240,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->pm.voltage_get          = nouveau_voltage_gpio_get;
                engine->pm.voltage_set          = nouveau_voltage_gpio_set;
                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                break;
        case 0x40:
@@ -289,6 +293,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->pm.voltage_set          = nouveau_voltage_gpio_set;
                engine->pm.temp_get             = nv40_temp_get;
                engine->vram.init               = nouveau_mem_detect;
+                engine->vram.takedown           = nouveau_stub_takedown;
                engine->vram.flags_valid        = nouveau_mem_flags_valid;
                break;
        case 0x50:
@@ -366,6 +371,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                else
                        engine->pm.temp_get     = nv40_temp_get;
                engine->vram.init               = nv50_vram_init;
+                engine->vram.takedown           = nv50_vram_fini;
                engine->vram.get                = nv50_vram_new;
                engine->vram.put                = nv50_vram_del;
                engine->vram.flags_valid        = nv50_vram_flags_valid;
@@ -411,9 +417,11 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
                engine->gpio.irq_unregister     = nv50_gpio_irq_unregister;
                engine->gpio.irq_enable         = nv50_gpio_irq_enable;
                engine->vram.init               = nvc0_vram_init;
+                engine->vram.takedown           = nv50_vram_fini;
                engine->vram.get                = nvc0_vram_new;
                engine->vram.put                = nv50_vram_del;
                engine->vram.flags_valid        = nvc0_vram_flags_valid;
+                engine->pm.temp_get             = nv84_temp_get;
                break;
        default:
                NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset);
@@ -447,8 +455,8 @@ nouveau_card_init_channel(struct drm_device *dev)
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        int ret;
-        ret = nouveau_channel_alloc(dev, &dev_priv->channel,
+        ret = nouveau_channel_alloc(dev, &dev_priv->channel, NULL,
-                                    (struct drm_file *)-2, NvDmaFB, NvDmaTT);
+                                    NvDmaFB, NvDmaTT);
        if (ret)
                return ret;
@@ -527,7 +535,7 @@ nouveau_card_init(struct drm_device *dev)
        nouveau_pm_init(dev);
-        ret = nouveau_mem_vram_init(dev);
+        ret = engine->vram.init(dev);
        if (ret)
                goto out_bios;
@@ -539,10 +547,14 @@ nouveau_card_init(struct drm_device *dev)
        if (ret)
                goto out_gpuobj;
-        ret = nouveau_mem_gart_init(dev);
+        ret = nouveau_mem_vram_init(dev);
        if (ret)
                goto out_instmem;
+        ret = nouveau_mem_gart_init(dev);
+        if (ret)
+                goto out_ttmvram;
        /* PMC */
        ret = engine->mc.init(dev);
        if (ret)
@@ -563,7 +575,7 @@ nouveau_card_init(struct drm_device *dev)
        if (ret)
                goto out_timer;
-        if (!nouveau_noaccel) {
+        if (!dev_priv->noaccel) {
                switch (dev_priv->card_type) {
                case NV_04:
                        nv04_graph_create(dev);
@@ -675,10 +687,10 @@ out_vblank:
        drm_vblank_cleanup(dev);
        engine->display.destroy(dev);
 out_fifo:
-        if (!nouveau_noaccel)
+        if (!dev_priv->noaccel)
                engine->fifo.takedown(dev);
 out_engine:
-        if (!nouveau_noaccel) {
+        if (!dev_priv->noaccel) {
                for (e = e - 1; e >= 0; e--) {
                        if (!dev_priv->eng[e])
                                continue;
@@ -696,12 +708,14 @@ out_mc:
        engine->mc.takedown(dev);
 out_gart:
        nouveau_mem_gart_fini(dev);
+out_ttmvram:
+        nouveau_mem_vram_fini(dev);
 out_instmem:
        engine->instmem.takedown(dev);
 out_gpuobj:
        nouveau_gpuobj_takedown(dev);
 out_vram:
-        nouveau_mem_vram_fini(dev);
+        engine->vram.takedown(dev);
 out_bios:
        nouveau_pm_fini(dev);
        nouveau_bios_takedown(dev);
@@ -718,12 +732,17 @@ static void nouveau_card_takedown(struct drm_device *dev)
        struct nouveau_engine *engine = &dev_priv->engine;
        int e;
+        drm_kms_helper_poll_fini(dev);
+        nouveau_fbcon_fini(dev);
        if (dev_priv->channel) {
-                nouveau_fence_fini(dev);
                nouveau_channel_put_unlocked(&dev_priv->channel);
+                nouveau_fence_fini(dev);
        }
-        if (!nouveau_noaccel) {
+        engine->display.destroy(dev);
+        if (!dev_priv->noaccel) {
                engine->fifo.takedown(dev);
                for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
                        if (dev_priv->eng[e]) {
@@ -748,10 +767,11 @@ static void nouveau_card_takedown(struct drm_device *dev)
        ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_TT);
        mutex_unlock(&dev->struct_mutex);
        nouveau_mem_gart_fini(dev);
+        nouveau_mem_vram_fini(dev);
        engine->instmem.takedown(dev);
        nouveau_gpuobj_takedown(dev);
-        nouveau_mem_vram_fini(dev);
+        engine->vram.takedown(dev);
        nouveau_irq_fini(dev);
        drm_vblank_cleanup(dev);
@@ -762,6 +782,41 @@ static void nouveau_card_takedown(struct drm_device *dev)
        vga_client_register(dev->pdev, NULL, NULL, NULL);
 }
+int
+nouveau_open(struct drm_device *dev, struct drm_file *file_priv)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fpriv *fpriv;
+        int ret;
+        fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
+        if (unlikely(!fpriv))
+                return -ENOMEM;
+        spin_lock_init(&fpriv->lock);
+        INIT_LIST_HEAD(&fpriv->channels);
+        if (dev_priv->card_type == NV_50) {
+                ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
+                                     &fpriv->vm);
+                if (ret) {
+                        kfree(fpriv);
+                        return ret;
+                }
+        } else
+        if (dev_priv->card_type >= NV_C0) {
+                ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL,
+                                     &fpriv->vm);
+                if (ret) {
+                        kfree(fpriv);
+                        return ret;
+                }
+        }
+        file_priv->driver_priv = fpriv;
+        return 0;
+}
 /* here a client dies, release the stuff that was allocated for its
 * file_priv */
 void nouveau_preclose(struct drm_device *dev, struct drm_file *file_priv)
@@ -769,6 +824,14 @@ void nouveau_preclose(struct drm_device *dev, struct drm_file *file_priv)
        nouveau_channel_cleanup(dev, file_priv);
 }
+void
+nouveau_postclose(struct drm_device *dev, struct drm_file *file_priv)
+{
+        struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+        nouveau_vm_ref(NULL, &fpriv->vm, NULL);
+        kfree(fpriv);
+}
 /* first module load, setup the mmio/fb mapping */
 /* KMS: we need mmio at load time, not when the first drm client opens. */
 int nouveau_firstopen(struct drm_device *dev)
@@ -933,6 +996,25 @@ int nouveau_load(struct drm_device *dev, unsigned long flags)
        NV_INFO(dev, "Detected an NV%2x generation card (0x%08x)\n",
                dev_priv->card_type, reg0);
+        /* Determine whether we'll attempt acceleration or not, some
+         * cards are disabled by default here due to them being known
+         * non-functional, or never been tested due to lack of hw.
+         */
+        dev_priv->noaccel = !!nouveau_noaccel;
+        if (nouveau_noaccel == -1) {
+                switch (dev_priv->chipset) {
+                case 0xc1: /* known broken */
+                case 0xc8: /* never tested */
+                        NV_INFO(dev, "acceleration disabled by default, pass "
+                                     "noaccel=0 to force enable\n");
+                        dev_priv->noaccel = true;
+                        break;
+                default:
+                        dev_priv->noaccel = false;
+                        break;
+                }
+        }
        ret = nouveau_remove_conflicting_drivers(dev);
        if (ret)
                goto err_mmio;
@@ -997,11 +1079,7 @@ void nouveau_lastclose(struct drm_device *dev)
 int nouveau_unload(struct drm_device *dev)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct nouveau_engine *engine = &dev_priv->engine;
-        drm_kms_helper_poll_fini(dev);
-        nouveau_fbcon_fini(dev);
-        engine->display.destroy(dev);
        nouveau_card_takedown(dev);
        iounmap(dev_priv->mmio);
@@ -1031,7 +1109,7 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
        case NOUVEAU_GETPARAM_BUS_TYPE:
                if (drm_pci_device_is_agp(dev))
                        getparam->value = NV_AGP;
-                else if (drm_pci_device_is_pcie(dev))
+                else if (pci_is_pcie(dev->pdev))
                        getparam->value = NV_PCIE;
                else
                        getparam->value = NV_PCI;
diff --git a/drivers/gpu/drm/nouveau/nouveau_temp.c b/drivers/gpu/drm/nouveau/nouveau_temp.c
index 649b0413b09f..081ca7b03e8a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_temp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_temp.c
@@ -43,7 +43,7 @@ nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
        /* Set the default sensor's contants */
        sensor->offset_constant = 0;
-        sensor->offset_mult = 1;
+        sensor->offset_mult = 0;
        sensor->offset_div = 1;
        sensor->slope_mult = 1;
        sensor->slope_div = 1;
@@ -99,6 +99,13 @@ nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
                        sensor->slope_mult = 431;
                        sensor->slope_div = 10000;
                        break;
+                case 0x67:
+                        sensor->offset_mult = -26149;
+                        sensor->offset_div = 100;
+                        sensor->slope_mult = 484;
+                        sensor->slope_div = 10000;
+                        break;
                }
        }
@@ -109,7 +116,7 @@ nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
        /* Read the entries from the table */
        for (i = 0; i < entries; i++) {
-                u16 value = ROM16(temp[1]);
+                s16 value = ROM16(temp[1]);
                switch (temp[0]) {
                case 0x01:
@@ -160,8 +167,8 @@ nv40_sensor_setup(struct drm_device *dev)
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
        struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
-        u32 offset = sensor->offset_mult / sensor->offset_div;
+        s32 offset = sensor->offset_mult / sensor->offset_div;
-        u32 sensor_calibration;
+        s32 sensor_calibration;
        /* set up the sensors */
        sensor_calibration = 120 - offset - sensor->offset_constant;
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.c b/drivers/gpu/drm/nouveau/nouveau_vm.c
index 519a6b4bba46..244fd38fdb84 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.c
@@ -369,23 +369,26 @@ nouveau_vm_link(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
 }
 static void
-nouveau_vm_unlink(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
+nouveau_vm_unlink(struct nouveau_vm *vm, struct nouveau_gpuobj *mpgd)
 {
        struct nouveau_vm_pgd *vpgd, *tmp;
+        struct nouveau_gpuobj *pgd = NULL;
-        if (!pgd)
+        if (!mpgd)
                return;
        mutex_lock(&vm->mm->mutex);
        list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
-                if (vpgd->obj != pgd)
+                if (vpgd->obj == mpgd) {
-                        continue;
+                        pgd = vpgd->obj;
+                        list_del(&vpgd->head);
-                list_del(&vpgd->head);
+                        kfree(vpgd);
-                nouveau_gpuobj_ref(NULL, &vpgd->obj);
+                        break;
-                kfree(vpgd);
+                }
        }
        mutex_unlock(&vm->mm->mutex);
+        nouveau_gpuobj_ref(NULL, &pgd);
 }
 static void
@@ -396,8 +399,8 @@ nouveau_vm_del(struct nouveau_vm *vm)
        list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
                nouveau_vm_unlink(vm, vpgd->obj);
        }
-        WARN_ON(nouveau_mm_fini(&vm->mm) != 0);
+        nouveau_mm_fini(&vm->mm);
        kfree(vm->pgt);
        kfree(vm);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.h b/drivers/gpu/drm/nouveau/nouveau_vm.h
index c48a9fc2b47b..579ca8cc223c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.h
@@ -41,6 +41,8 @@ struct nouveau_vm_pgd {
 };
 struct nouveau_vma {
+        struct list_head head;
+        int refcount;
        struct nouveau_vm *vm;
        struct nouveau_mm_node *node;
        u64 offset;
diff --git a/drivers/gpu/drm/nouveau/nv04_crtc.c b/drivers/gpu/drm/nouveau/nv04_crtc.c
index f1a3ae491995..118261d4927a 100644
--- a/drivers/gpu/drm/nouveau/nv04_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv04_crtc.c
@@ -1035,7 +1035,7 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
        drm_crtc_helper_add(&nv_crtc->base, &nv04_crtc_helper_funcs);
        drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
-        ret = nouveau_bo_new(dev, NULL, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
+        ret = nouveau_bo_new(dev, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
                             0, 0x0000, &nv_crtc->cursor.nvbo);
        if (!ret) {
                ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
diff --git a/drivers/gpu/drm/nouveau/nv04_instmem.c b/drivers/gpu/drm/nouveau/nv04_instmem.c
index b8611b955313..c1248e0740a3 100644
--- a/drivers/gpu/drm/nouveau/nv04_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv04_instmem.c
@@ -28,6 +28,31 @@ int nv04_instmem_init(struct drm_device *dev)
        /* RAMIN always available */
        dev_priv->ramin_available = true;
+        /* Reserve space at end of VRAM for PRAMIN */
+        if (dev_priv->card_type >= NV_40) {
+                u32 vs = hweight8((nv_rd32(dev, 0x001540) & 0x0000ff00) >> 8);
+                u32 rsvd;
+                /* estimate grctx size, the magics come from nv40_grctx.c */
+                if      (dev_priv->chipset == 0x40) rsvd = 0x6aa0 * vs;
+                else if (dev_priv->chipset  < 0x43) rsvd = 0x4f00 * vs;
+                else if (nv44_graph_class(dev))     rsvd = 0x4980 * vs;
+                else                                rsvd = 0x4a40 * vs;
+                rsvd += 16 * 1024;
+                rsvd *= dev_priv->engine.fifo.channels;
+                /* pciegart table */
+                if (pci_is_pcie(dev->pdev))
+                        rsvd += 512 * 1024;
+                /* object storage */
+                rsvd += 512 * 1024;
+                dev_priv->ramin_rsvd_vram = round_up(rsvd, 4096);
+        } else {
+                dev_priv->ramin_rsvd_vram = 512 * 1024;
+        }
        /* Setup shared RAMHT */
        ret = nouveau_gpuobj_new_fake(dev, 0x10000, ~0, 4096,
                                      NVOBJ_FLAG_ZERO_ALLOC, &ramht);
@@ -112,7 +137,8 @@ nv04_instmem_resume(struct drm_device *dev)
 }
 int
-nv04_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
+nv04_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
+                 u32 size, u32 align)
 {
        struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
        struct drm_mm_node *ramin = NULL;
diff --git a/drivers/gpu/drm/nouveau/nv50_crtc.c b/drivers/gpu/drm/nouveau/nv50_crtc.c
index ebabacf38da9..46ad59ea2185 100644
--- a/drivers/gpu/drm/nouveau/nv50_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv50_crtc.c
@@ -104,7 +104,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
                OUT_RING(evo, nv_crtc->lut.depth == 8 ?
                                NV50_EVO_CRTC_CLUT_MODE_OFF :
                                NV50_EVO_CRTC_CLUT_MODE_ON);
-                OUT_RING(evo, (nv_crtc->lut.nvbo->bo.mem.start << PAGE_SHIFT) >> 8);
+                OUT_RING(evo, nv_crtc->lut.nvbo->bo.offset >> 8);
                if (dev_priv->chipset != 0x50) {
                        BEGIN_RING(evo, 0, NV84_EVO_CRTC(index, CLUT_DMA), 1);
                        OUT_RING(evo, NvEvoVRAM);
@@ -372,7 +372,7 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
        nouveau_bo_unmap(cursor);
-        nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT);
+        nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset);
        nv_crtc->cursor.show(nv_crtc, true);
 out:
@@ -546,7 +546,7 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
                }
        }
-        nv_crtc->fb.offset = fb->nvbo->bo.mem.start << PAGE_SHIFT;
+        nv_crtc->fb.offset = fb->nvbo->bo.offset;
        nv_crtc->fb.tile_flags = nouveau_bo_tile_layout(fb->nvbo);
        nv_crtc->fb.cpp = drm_fb->bits_per_pixel / 8;
        if (!nv_crtc->fb.blanked && dev_priv->chipset != 0x50) {
@@ -747,7 +747,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
        }
        nv_crtc->lut.depth = 0;
-        ret = nouveau_bo_new(dev, NULL, 4096, 0x100, TTM_PL_FLAG_VRAM,
+        ret = nouveau_bo_new(dev, 4096, 0x100, TTM_PL_FLAG_VRAM,
                             0, 0x0000, &nv_crtc->lut.nvbo);
        if (!ret) {
                ret = nouveau_bo_pin(nv_crtc->lut.nvbo, TTM_PL_FLAG_VRAM);
@@ -773,7 +773,7 @@ nv50_crtc_create(struct drm_device *dev, int index)
        drm_crtc_helper_add(&nv_crtc->base, &nv50_crtc_helper_funcs);
        drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
-        ret = nouveau_bo_new(dev, NULL, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
+        ret = nouveau_bo_new(dev, 64*64*4, 0x100, TTM_PL_FLAG_VRAM,
                             0, 0x0000, &nv_crtc->cursor.nvbo);
        if (!ret) {
                ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 08da478ba544..db1a5f4b711d 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -415,8 +415,6 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
        /* synchronise with the rendering channel, if necessary */
        if (likely(chan)) {
-                u64 offset = dispc->sem.bo->vma.offset + dispc->sem.offset;
                ret = RING_SPACE(chan, 10);
                if (ret) {
                        WIND_RING(evo);
@@ -438,6 +436,8 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                        else
                                OUT_RING  (chan, chan->vram_handle);
                } else {
+                        u64 offset = chan->dispc_vma[nv_crtc->index].offset;
+                        offset += dispc->sem.offset;
                        BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0010, 4);
                        OUT_RING  (chan, upper_32_bits(offset));
                        OUT_RING  (chan, lower_32_bits(offset));
@@ -484,7 +484,7 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
        OUT_RING  (evo, 0x00000000);
        OUT_RING  (evo, 0x00000000);
        BEGIN_RING(evo, 0, 0x0800, 5);
-        OUT_RING  (evo, (nv_fb->nvbo->bo.mem.start << PAGE_SHIFT) >> 8);
+        OUT_RING  (evo, nv_fb->nvbo->bo.offset >> 8);
        OUT_RING  (evo, 0);
        OUT_RING  (evo, (fb->height << 16) | fb->width);
        OUT_RING  (evo, nv_fb->r_pitch);
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.c b/drivers/gpu/drm/nouveau/nv50_evo.c
index c8e83c1a4de8..c99d9751880c 100644
--- a/drivers/gpu/drm/nouveau/nv50_evo.c
+++ b/drivers/gpu/drm/nouveau/nv50_evo.c
@@ -38,6 +38,7 @@ nv50_evo_channel_del(struct nouveau_channel **pevo)
                return;
        *pevo = NULL;
+        nouveau_ramht_ref(NULL, &evo->ramht, evo);
        nouveau_gpuobj_channel_takedown(evo);
        nouveau_bo_unmap(evo->pushbuf_bo);
        nouveau_bo_ref(NULL, &evo->pushbuf_bo);
@@ -116,7 +117,7 @@ nv50_evo_channel_new(struct drm_device *dev, int chid,
        evo->user_get = 4;
        evo->user_put = 0;
-        ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
+        ret = nouveau_bo_new(dev, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
                             &evo->pushbuf_bo);
        if (ret == 0)
                ret = nouveau_bo_pin(evo->pushbuf_bo, TTM_PL_FLAG_VRAM);
@@ -153,7 +154,7 @@ nv50_evo_channel_init(struct nouveau_channel *evo)
 {
        struct drm_device *dev = evo->dev;
        int id = evo->id, ret, i;
-        u64 pushbuf = evo->pushbuf_bo->bo.mem.start << PAGE_SHIFT;
+        u64 pushbuf = evo->pushbuf_bo->bo.offset;
        u32 tmp;
        tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
@@ -331,16 +332,15 @@ nv50_evo_create(struct drm_device *dev)
                if (ret)
                        goto err;
-                ret = nouveau_bo_new(dev, NULL, 4096, 0x1000, TTM_PL_FLAG_VRAM,
+                ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
                                     0, 0x0000, &dispc->sem.bo);
                if (!ret) {
-                        offset = dispc->sem.bo->bo.mem.start << PAGE_SHIFT;
                        ret = nouveau_bo_pin(dispc->sem.bo, TTM_PL_FLAG_VRAM);
                        if (!ret)
                                ret = nouveau_bo_map(dispc->sem.bo);
                        if (ret)
                                nouveau_bo_ref(NULL, &dispc->sem.bo);
+                        offset = dispc->sem.bo->bo.offset;
                }
                if (ret)
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index 791ded1c5c6d..dc75a7206524 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -159,7 +159,7 @@ nv50_fbcon_accel_init(struct fb_info *info)
        struct drm_device *dev = nfbdev->dev;
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_channel *chan = dev_priv->channel;
-        struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
+        struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
        int ret, format;
        switch (info->var.bits_per_pixel) {
@@ -247,8 +247,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
        OUT_RING(chan, info->fix.line_length);
        OUT_RING(chan, info->var.xres_virtual);
        OUT_RING(chan, info->var.yres_virtual);
-        OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, upper_32_bits(fb->vma.offset));
-        OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, lower_32_bits(fb->vma.offset));
        BEGIN_RING(chan, NvSub2D, 0x0230, 2);
        OUT_RING(chan, format);
        OUT_RING(chan, 1);
@@ -256,8 +256,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
        OUT_RING(chan, info->fix.line_length);
        OUT_RING(chan, info->var.xres_virtual);
        OUT_RING(chan, info->var.yres_virtual);
-        OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, upper_32_bits(fb->vma.offset));
-        OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING(chan, lower_32_bits(fb->vma.offset));
        return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c
index 4f95a1e5822e..a7c12c94a5a6 100644
--- a/drivers/gpu/drm/nouveau/nv50_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv50_instmem.c
@@ -305,9 +305,9 @@ struct nv50_gpuobj_node {
        u32 align;
 };
 int
-nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
+nv50_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
+                 u32 size, u32 align)
 {
        struct drm_device *dev = gpuobj->dev;
        struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -336,7 +336,7 @@ nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
                if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER))
                        flags |= NV_MEM_ACCESS_SYS;
-                ret = nouveau_vm_get(dev_priv->chan_vm, size, 12, flags,
+                ret = nouveau_vm_get(chan->vm, size, 12, flags,
                                     &node->chan_vma);
                if (ret) {
                        vram->put(dev, &node->vram);
@@ -345,7 +345,7 @@ nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
                }
                nouveau_vm_map(&node->chan_vma, node->vram);
-                gpuobj->vinst = node->chan_vma.offset;
+                gpuobj->linst = node->chan_vma.offset;
        }
        gpuobj->size = size;
diff --git a/drivers/gpu/drm/nouveau/nv50_vm.c b/drivers/gpu/drm/nouveau/nv50_vm.c
index 1a0dd491a0e4..40b84f22d819 100644
--- a/drivers/gpu/drm/nouveau/nv50_vm.c
+++ b/drivers/gpu/drm/nouveau/nv50_vm.c
@@ -156,7 +156,7 @@ nv50_vm_flush(struct nouveau_vm *vm)
        pinstmem->flush(vm->dev);
        /* BAR */
-        if (vm != dev_priv->chan_vm) {
+        if (vm == dev_priv->bar1_vm || vm == dev_priv->bar3_vm) {
                nv50_vm_flush_engine(vm->dev, 6);
                return;
        }
diff --git a/drivers/gpu/drm/nouveau/nv50_vram.c b/drivers/gpu/drm/nouveau/nv50_vram.c
index ffbc3d8cf5be..af32daecd1ed 100644
--- a/drivers/gpu/drm/nouveau/nv50_vram.c
+++ b/drivers/gpu/drm/nouveau/nv50_vram.c
@@ -51,9 +51,7 @@ void
 nv50_vram_del(struct drm_device *dev, struct nouveau_mem **pmem)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        struct nouveau_mm *mm = dev_priv->engine.vram.mm;
-        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
-        struct nouveau_mm *mm = man->priv;
        struct nouveau_mm_node *this;
        struct nouveau_mem *mem;
@@ -84,9 +82,7 @@ nv50_vram_new(struct drm_device *dev, u64 size, u32 align, u32 size_nc,
              u32 memtype, struct nouveau_mem **pmem)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        struct nouveau_mm *mm = dev_priv->engine.vram.mm;
-        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
-        struct nouveau_mm *mm = man->priv;
        struct nouveau_mm_node *r;
        struct nouveau_mem *mem;
        int comp = (memtype & 0x300) >> 8;
@@ -190,22 +186,35 @@ int
 nv50_vram_init(struct drm_device *dev)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+        const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+        u32 rblock, length;
        dev_priv->vram_size  = nv_rd32(dev, 0x10020c);
        dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
        dev_priv->vram_size &= 0xffffffff00ULL;
-        switch (dev_priv->chipset) {
+        /* IGPs, no funky reordering happens here, they don't have VRAM */
-        case 0xaa:
+        if (dev_priv->chipset == 0xaa ||
-        case 0xac:
+            dev_priv->chipset == 0xac ||
-        case 0xaf:
+            dev_priv->chipset == 0xaf) {
                dev_priv->vram_sys_base = (u64)nv_rd32(dev, 0x100e10) << 12;
-                dev_priv->vram_rblock_size = 4096;
+                rblock = 4096 >> 12;
-                break;
+        } else {
-        default:
+                rblock = nv50_vram_rblock(dev) >> 12;
-                dev_priv->vram_rblock_size = nv50_vram_rblock(dev);
-                break;
        }
-        return 0;
+        length = (dev_priv->vram_size >> 12) - rsvd_head - rsvd_tail;
+        return nouveau_mm_init(&vram->mm, rsvd_head, length, rblock);
+}
+void
+nv50_vram_fini(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        nouveau_mm_fini(&vram->mm);
 }
diff --git a/drivers/gpu/drm/nouveau/nvc0_copy.c b/drivers/gpu/drm/nouveau/nvc0_copy.c
index 208fa7ab3f42..5ebcd74244db 100644
--- a/drivers/gpu/drm/nouveau/nvc0_copy.c
+++ b/drivers/gpu/drm/nouveau/nvc0_copy.c
@@ -48,14 +48,14 @@ nvc0_copy_context_new(struct nouveau_channel *chan, int engine)
        struct nouveau_gpuobj *ctx = NULL;
        int ret;
-        ret = nouveau_gpuobj_new(dev, NULL, 256, 256,
+        ret = nouveau_gpuobj_new(dev, chan, 256, 256,
                                 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER |
                                 NVOBJ_FLAG_ZERO_ALLOC, &ctx);
        if (ret)
                return ret;
-        nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(ctx->vinst));
+        nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(ctx->linst));
-        nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(ctx->vinst));
+        nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(ctx->linst));
        dev_priv->engine.instmem.flush(dev);
        chan->engctx[engine] = ctx;
diff --git a/drivers/gpu/drm/nouveau/nvc0_fb.c b/drivers/gpu/drm/nouveau/nvc0_fb.c
index 26a996025dd2..08e6b118f021 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fb.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2010 Red Hat Inc.
+ * Copyright 2011 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
@@ -23,16 +23,80 @@
 */
 #include "drmP.h"
+#include "drm.h"
 #include "nouveau_drv.h"
+#include "nouveau_drm.h"
+struct nvc0_fb_priv {
+        struct page *r100c10_page;
+        dma_addr_t r100c10;
+};
+static void
+nvc0_fb_destroy(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nvc0_fb_priv *priv = pfb->priv;
+        if (priv->r100c10_page) {
+                pci_unmap_page(dev->pdev, priv->r100c10, PAGE_SIZE,
+                               PCI_DMA_BIDIRECTIONAL);
+                __free_page(priv->r100c10_page);
+        }
+        kfree(priv);
+        pfb->priv = NULL;
+}
+static int
+nvc0_fb_create(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
+        struct nvc0_fb_priv *priv;
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+        pfb->priv = priv;
+        priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+        if (!priv->r100c10_page) {
+                nvc0_fb_destroy(dev);
+                return -ENOMEM;
+        }
+        priv->r100c10 = pci_map_page(dev->pdev, priv->r100c10_page, 0,
+                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        if (pci_dma_mapping_error(dev->pdev, priv->r100c10)) {
+                nvc0_fb_destroy(dev);
+                return -EFAULT;
+        }
+        return 0;
+}
 int
 nvc0_fb_init(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nvc0_fb_priv *priv;
+        int ret;
+        if (!dev_priv->engine.fb.priv) {
+                ret = nvc0_fb_create(dev);
+                if (ret)
+                        return ret;
+        }
+        priv = dev_priv->engine.fb.priv;
+        nv_wr32(dev, 0x100c10, priv->r100c10 >> 8);
        return 0;
 }
 void
 nvc0_fb_takedown(struct drm_device *dev)
 {
+        nvc0_fb_destroy(dev);
 }
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
index fa5d4c234383..a495e48197ca 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -159,7 +159,7 @@ nvc0_fbcon_accel_init(struct fb_info *info)
        struct drm_device *dev = nfbdev->dev;
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nouveau_channel *chan = dev_priv->channel;
-        struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
+        struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
        int ret, format;
        ret = nouveau_gpuobj_gr_new(chan, 0x902d, 0x902d);
@@ -203,8 +203,8 @@ nvc0_fbcon_accel_init(struct fb_info *info)
        BEGIN_NVC0(chan, 2, NvSub2D, 0x0000, 1);
        OUT_RING  (chan, 0x0000902d);
        BEGIN_NVC0(chan, 2, NvSub2D, 0x0104, 2);
-        OUT_RING  (chan, upper_32_bits(chan->notifier_bo->bo.offset));
+        OUT_RING  (chan, upper_32_bits(chan->notifier_vma.offset));
-        OUT_RING  (chan, lower_32_bits(chan->notifier_bo->bo.offset));
+        OUT_RING  (chan, lower_32_bits(chan->notifier_vma.offset));
        BEGIN_NVC0(chan, 2, NvSub2D, 0x0290, 1);
        OUT_RING  (chan, 0);
        BEGIN_NVC0(chan, 2, NvSub2D, 0x0888, 1);
@@ -249,8 +249,8 @@ nvc0_fbcon_accel_init(struct fb_info *info)
        OUT_RING  (chan, info->fix.line_length);
        OUT_RING  (chan, info->var.xres_virtual);
        OUT_RING  (chan, info->var.yres_virtual);
-        OUT_RING  (chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, upper_32_bits(fb->vma.offset));
-        OUT_RING  (chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, lower_32_bits(fb->vma.offset));
        BEGIN_NVC0(chan, 2, NvSub2D, 0x0230, 10);
        OUT_RING  (chan, format);
        OUT_RING  (chan, 1);
@@ -260,8 +260,8 @@ nvc0_fbcon_accel_init(struct fb_info *info)
        OUT_RING  (chan, info->fix.line_length);
        OUT_RING  (chan, info->var.xres_virtual);
        OUT_RING  (chan, info->var.yres_virtual);
-        OUT_RING  (chan, upper_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, upper_32_bits(fb->vma.offset));
-        OUT_RING  (chan, lower_32_bits(nvbo->vma.offset));
+        OUT_RING  (chan, lower_32_bits(fb->vma.offset));
        FIRE_RING (chan);
        return 0;
diff --git a/drivers/gpu/drm/nouveau/nvc0_fifo.c b/drivers/gpu/drm/nouveau/nvc0_fifo.c
index fb4f5943e01b..6f9f341c3e86 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fifo.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fifo.c
@@ -210,10 +210,10 @@ nvc0_fifo_unload_context(struct drm_device *dev)
        int i;
        for (i = 0; i < 128; i++) {
-                if (!(nv_rd32(dev, 0x003004 + (i * 4)) & 1))
+                if (!(nv_rd32(dev, 0x003004 + (i * 8)) & 1))
                        continue;
-                nv_mask(dev, 0x003004 + (i * 4), 0x00000001, 0x00000000);
+                nv_mask(dev, 0x003004 + (i * 8), 0x00000001, 0x00000000);
                nv_wr32(dev, 0x002634, i);
                if (!nv_wait(dev, 0x002634, 0xffffffff, i)) {
                        NV_INFO(dev, "PFIFO: kick ch %d failed: 0x%08x\n",
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.c b/drivers/gpu/drm/nouveau/nvc0_graph.c
index ca6db204d644..3a97431996c5 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.c
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.c
@@ -28,7 +28,34 @@
 #include "nouveau_drv.h"
 #include "nouveau_mm.h"
 #include "nvc0_graph.h"
+#include "nvc0_grhub.fuc.h"
+#include "nvc0_grgpc.fuc.h"
+static void
+nvc0_graph_ctxctl_debug_unit(struct drm_device *dev, u32 base)
+{
+        NV_INFO(dev, "PGRAPH: %06x - done 0x%08x\n", base,
+                nv_rd32(dev, base + 0x400));
+        NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
+                nv_rd32(dev, base + 0x800), nv_rd32(dev, base + 0x804),
+                nv_rd32(dev, base + 0x808), nv_rd32(dev, base + 0x80c));
+        NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
+                nv_rd32(dev, base + 0x810), nv_rd32(dev, base + 0x814),
+                nv_rd32(dev, base + 0x818), nv_rd32(dev, base + 0x81c));
+}
+static void
+nvc0_graph_ctxctl_debug(struct drm_device *dev)
+{
+        u32 gpcnr = nv_rd32(dev, 0x409604) & 0xffff;
+        u32 gpc;
+        nvc0_graph_ctxctl_debug_unit(dev, 0x409000);
+        for (gpc = 0; gpc < gpcnr; gpc++)
+                nvc0_graph_ctxctl_debug_unit(dev, 0x502000 + (gpc * 0x8000));
+}
 static int
 nvc0_graph_load_context(struct nouveau_channel *chan)
@@ -72,24 +99,44 @@ nvc0_graph_construct_context(struct nouveau_channel *chan)
        if (!ctx)
                return -ENOMEM;
-        nvc0_graph_load_context(chan);
+        if (!nouveau_ctxfw) {
+                nv_wr32(dev, 0x409840, 0x80000000);
-        nv_wo32(grch->grctx, 0x1c, 1);
+                nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
-        nv_wo32(grch->grctx, 0x20, 0);
+                nv_wr32(dev, 0x409504, 0x00000001);
-        nv_wo32(grch->grctx, 0x28, 0);
+                if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
-        nv_wo32(grch->grctx, 0x2c, 0);
+                        NV_ERROR(dev, "PGRAPH: HUB_SET_CHAN timeout\n");
-        dev_priv->engine.instmem.flush(dev);
+                        nvc0_graph_ctxctl_debug(dev);
+                        ret = -EBUSY;
-        ret = nvc0_grctx_generate(chan);
+                        goto err;
-        if (ret) {
+                }
-                kfree(ctx);
+        } else {
-                return ret;
+                nvc0_graph_load_context(chan);
+                nv_wo32(grch->grctx, 0x1c, 1);
+                nv_wo32(grch->grctx, 0x20, 0);
+                nv_wo32(grch->grctx, 0x28, 0);
+                nv_wo32(grch->grctx, 0x2c, 0);
+                dev_priv->engine.instmem.flush(dev);
        }
-        ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
+        ret = nvc0_grctx_generate(chan);
-        if (ret) {
+        if (ret)
-                kfree(ctx);
+                goto err;
-                return ret;
+        if (!nouveau_ctxfw) {
+                nv_wr32(dev, 0x409840, 0x80000000);
+                nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
+                nv_wr32(dev, 0x409504, 0x00000002);
+                if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
+                        NV_ERROR(dev, "PGRAPH: HUB_CTX_SAVE timeout\n");
+                        nvc0_graph_ctxctl_debug(dev);
+                        ret = -EBUSY;
+                        goto err;
+                }
+        } else {
+                ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
+                if (ret)
+                        goto err;
        }
        for (i = 0; i < priv->grctx_size; i += 4)
@@ -97,6 +144,10 @@ nvc0_graph_construct_context(struct nouveau_channel *chan)
        priv->grctx_vals = ctx;
        return 0;
+err:
+        kfree(ctx);
+        return ret;
 }
 static int
@@ -108,50 +159,50 @@ nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
        int i = 0, gpc, tp, ret;
        u32 magic;
-        ret = nouveau_gpuobj_new(dev, NULL, 0x2000, 256, NVOBJ_FLAG_VM,
+        ret = nouveau_gpuobj_new(dev, chan, 0x2000, 256, NVOBJ_FLAG_VM,
                                 &grch->unk408004);
        if (ret)
                return ret;
-        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 256, NVOBJ_FLAG_VM,
+        ret = nouveau_gpuobj_new(dev, chan, 0x8000, 256, NVOBJ_FLAG_VM,
                                 &grch->unk40800c);
        if (ret)
                return ret;
-        ret = nouveau_gpuobj_new(dev, NULL, 384 * 1024, 4096,
+        ret = nouveau_gpuobj_new(dev, chan, 384 * 1024, 4096,
                                 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
                                 &grch->unk418810);
        if (ret)
                return ret;
-        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0, NVOBJ_FLAG_VM,
+        ret = nouveau_gpuobj_new(dev, chan, 0x1000, 0, NVOBJ_FLAG_VM,
                                 &grch->mmio);
        if (ret)
                return ret;
        nv_wo32(grch->mmio, i++ * 4, 0x00408004);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
        nv_wo32(grch->mmio, i++ * 4, 0x00408008);
        nv_wo32(grch->mmio, i++ * 4, 0x80000018);
        nv_wo32(grch->mmio, i++ * 4, 0x0040800c);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
        nv_wo32(grch->mmio, i++ * 4, 0x00408010);
        nv_wo32(grch->mmio, i++ * 4, 0x80000000);
        nv_wo32(grch->mmio, i++ * 4, 0x00418810);
-        nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->vinst >> 12);
+        nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->linst >> 12);
        nv_wo32(grch->mmio, i++ * 4, 0x00419848);
-        nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->vinst >> 12);
+        nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->linst >> 12);
        nv_wo32(grch->mmio, i++ * 4, 0x00419004);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
        nv_wo32(grch->mmio, i++ * 4, 0x00419008);
        nv_wo32(grch->mmio, i++ * 4, 0x00000000);
        nv_wo32(grch->mmio, i++ * 4, 0x00418808);
-        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
+        nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
        nv_wo32(grch->mmio, i++ * 4, 0x0041880c);
        nv_wo32(grch->mmio, i++ * 4, 0x80000018);
@@ -159,7 +210,7 @@ nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
        nv_wo32(grch->mmio, i++ * 4, 0x00405830);
        nv_wo32(grch->mmio, i++ * 4, magic);
        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-                for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x02fc) {
+                for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x0324) {
                        u32 reg = 0x504520 + (gpc * 0x8000) + (tp * 0x0800);
                        nv_wo32(grch->mmio, i++ * 4, reg);
                        nv_wo32(grch->mmio, i++ * 4, magic);
@@ -186,7 +237,7 @@ nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
                return -ENOMEM;
        chan->engctx[NVOBJ_ENGINE_GR] = grch;
-        ret = nouveau_gpuobj_new(dev, NULL, priv->grctx_size, 256,
+        ret = nouveau_gpuobj_new(dev, chan, priv->grctx_size, 256,
                                 NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
                                 &grch->grctx);
        if (ret)
@@ -197,8 +248,8 @@ nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
        if (ret)
                goto error;
-        nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->vinst) | 4);
+        nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->linst) | 4);
-        nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->vinst));
+        nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->linst));
        pinstmem->flush(dev);
        if (!priv->grctx_vals) {
@@ -210,15 +261,20 @@ nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
        for (i = 0; i < priv->grctx_size; i += 4)
                nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
-        nv_wo32(grctx, 0xf4, 0);
+        if (!nouveau_ctxfw) {
-        nv_wo32(grctx, 0xf8, 0);
+                nv_wo32(grctx, 0x00, grch->mmio_nr);
-        nv_wo32(grctx, 0x10, grch->mmio_nr);
+                nv_wo32(grctx, 0x04, grch->mmio->linst >> 8);
-        nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->vinst));
+        } else {
-        nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->vinst));
+                nv_wo32(grctx, 0xf4, 0);
-        nv_wo32(grctx, 0x1c, 1);
+                nv_wo32(grctx, 0xf8, 0);
-        nv_wo32(grctx, 0x20, 0);
+                nv_wo32(grctx, 0x10, grch->mmio_nr);
-        nv_wo32(grctx, 0x28, 0);
+                nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->linst));
-        nv_wo32(grctx, 0x2c, 0);
+                nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->linst));
+                nv_wo32(grctx, 0x1c, 1);
+                nv_wo32(grctx, 0x20, 0);
+                nv_wo32(grctx, 0x28, 0);
+                nv_wo32(grctx, 0x2c, 0);
+        }
        pinstmem->flush(dev);
        return 0;
@@ -296,6 +352,7 @@ static void
 nvc0_graph_init_gpc_0(struct drm_device *dev)
 {
        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
+        const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tp_total);
        u32 data[TP_MAX / 8];
        u8  tpnr[GPC_MAX];
        int i, gpc, tpc;
@@ -307,13 +364,6 @@ nvc0_graph_init_gpc_0(struct drm_device *dev)
         * 465: 3/4/4/0 4        7
         * 470: 3/3/4/4 5        5
         * 480: 3/4/4/4 6        6
-         *
-         * magicgpc918
-         * 450: 00200000 00000000001000000000000000000000
-         * 460: 00124925 00000000000100100100100100100101
-         * 465: 000ba2e9 00000000000010111010001011101001
-         * 470: 00092493 00000000000010010010010010010011
-         * 480: 00088889 00000000000010001000100010001001
         */
        memset(data, 0x00, sizeof(data));
@@ -336,10 +386,10 @@ nvc0_graph_init_gpc_0(struct drm_device *dev)
                nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
                                                  priv->tp_nr[gpc]);
                nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tp_total);
-                nv_wr32(dev, GPC_UNIT(gpc, 0x0918), priv->magicgpc918);
+                nv_wr32(dev, GPC_UNIT(gpc, 0x0918), magicgpc918);
        }
-        nv_wr32(dev, GPC_BCAST(0x1bd4), priv->magicgpc918);
+        nv_wr32(dev, GPC_BCAST(0x1bd4), magicgpc918);
        nv_wr32(dev, GPC_BCAST(0x08ac), priv->rop_nr);
 }
@@ -419,8 +469,51 @@ nvc0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
 static int
 nvc0_graph_init_ctxctl(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
        u32 r000260;
+        int i;
+        if (!nouveau_ctxfw) {
+                /* load HUB microcode */
+                r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
+                nv_wr32(dev, 0x4091c0, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
+                        nv_wr32(dev, 0x4091c4, nvc0_grhub_data[i]);
+                nv_wr32(dev, 0x409180, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
+                        if ((i & 0x3f) == 0)
+                                nv_wr32(dev, 0x409188, i >> 6);
+                        nv_wr32(dev, 0x409184, nvc0_grhub_code[i]);
+                }
+                /* load GPC microcode */
+                nv_wr32(dev, 0x41a1c0, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
+                        nv_wr32(dev, 0x41a1c4, nvc0_grgpc_data[i]);
+                nv_wr32(dev, 0x41a180, 0x01000000);
+                for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
+                        if ((i & 0x3f) == 0)
+                                nv_wr32(dev, 0x41a188, i >> 6);
+                        nv_wr32(dev, 0x41a184, nvc0_grgpc_code[i]);
+                }
+                nv_wr32(dev, 0x000260, r000260);
+                /* start HUB ucode running, it'll init the GPCs */
+                nv_wr32(dev, 0x409800, dev_priv->chipset);
+                nv_wr32(dev, 0x40910c, 0x00000000);
+                nv_wr32(dev, 0x409100, 0x00000002);
+                if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
+                        NV_ERROR(dev, "PGRAPH: HUB_INIT timed out\n");
+                        nvc0_graph_ctxctl_debug(dev);
+                        return -EBUSY;
+                }
+                priv->grctx_size = nv_rd32(dev, 0x409804);
+                return 0;
+        }
        /* load fuc microcode */
        r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
@@ -528,6 +621,22 @@ nvc0_graph_isr_chid(struct drm_device *dev, u64 inst)
 }
 static void
+nvc0_graph_ctxctl_isr(struct drm_device *dev)
+{
+        u32 ustat = nv_rd32(dev, 0x409c18);
+        if (ustat & 0x00000001)
+                NV_INFO(dev, "PGRAPH: CTXCTRL ucode error\n");
+        if (ustat & 0x00080000)
+                NV_INFO(dev, "PGRAPH: CTXCTRL watchdog timeout\n");
+        if (ustat & ~0x00080001)
+                NV_INFO(dev, "PGRAPH: CTXCTRL 0x%08x\n", ustat);
+        nvc0_graph_ctxctl_debug(dev);
+        nv_wr32(dev, 0x409c20, ustat);
+}
+static void
 nvc0_graph_isr(struct drm_device *dev)
 {
        u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
@@ -578,11 +687,7 @@ nvc0_graph_isr(struct drm_device *dev)
        }
        if (stat & 0x00080000) {
-                u32 ustat = nv_rd32(dev, 0x409c18);
+                nvc0_graph_ctxctl_isr(dev);
-                NV_INFO(dev, "PGRAPH: CTXCTRL ustat 0x%08x\n", ustat);
-                nv_wr32(dev, 0x409c20, ustat);
                nv_wr32(dev, 0x400100, 0x00080000);
                stat &= ~0x00080000;
        }
@@ -606,7 +711,7 @@ nvc0_runk140_isr(struct drm_device *dev)
                u32 st0 = nv_mask(dev, reg + 0x1020, 0, 0);
                u32 st1 = nv_mask(dev, reg + 0x1420, 0, 0);
-                NV_INFO(dev, "PRUNK140: %d 0x%08x 0x%08x\n", unit, st0, st1);
+                NV_DEBUG(dev, "PRUNK140: %d 0x%08x 0x%08x\n", unit, st0, st1);
                units &= ~(1 << unit);
        }
 }
@@ -651,10 +756,12 @@ nvc0_graph_destroy(struct drm_device *dev, int engine)
 {
        struct nvc0_graph_priv *priv = nv_engine(dev, engine);
-        nvc0_graph_destroy_fw(&priv->fuc409c);
+        if (nouveau_ctxfw) {
-        nvc0_graph_destroy_fw(&priv->fuc409d);
+                nvc0_graph_destroy_fw(&priv->fuc409c);
-        nvc0_graph_destroy_fw(&priv->fuc41ac);
+                nvc0_graph_destroy_fw(&priv->fuc409d);
-        nvc0_graph_destroy_fw(&priv->fuc41ad);
+                nvc0_graph_destroy_fw(&priv->fuc41ac);
+                nvc0_graph_destroy_fw(&priv->fuc41ad);
+        }
        nouveau_irq_unregister(dev, 12);
        nouveau_irq_unregister(dev, 25);
@@ -675,13 +782,10 @@ nvc0_graph_create(struct drm_device *dev)
        struct drm_nouveau_private *dev_priv = dev->dev_private;
        struct nvc0_graph_priv *priv;
        int ret, gpc, i;
+        u32 fermi;
-        switch (dev_priv->chipset) {
+        fermi = nvc0_graph_class(dev);
-        case 0xc0:
+        if (!fermi) {
-        case 0xc3:
-        case 0xc4:
-                break;
-        default:
                NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
                return 0;
        }
@@ -701,15 +805,17 @@ nvc0_graph_create(struct drm_device *dev)
        nouveau_irq_register(dev, 12, nvc0_graph_isr);
        nouveau_irq_register(dev, 25, nvc0_runk140_isr);
-        if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
+        if (nouveau_ctxfw) {
-            nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
+                NV_INFO(dev, "PGRAPH: using external firmware\n");
-            nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
+                if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
-            nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
+                    nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
-                ret = 0;
+                    nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
-                goto error;
+                    nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
+                        ret = 0;
+                        goto error;
+                }
        }
        ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
        if (ret)
                goto error;
@@ -735,25 +841,28 @@ nvc0_graph_create(struct drm_device *dev)
        case 0xc0:
                if (priv->tp_total == 11) { /* 465, 3/4/4/0, 4 */
                        priv->magic_not_rop_nr = 0x07;
-                        /* filled values up to tp_total, the rest 0 */
-                        priv->magicgpc918      = 0x000ba2e9;
                } else
                if (priv->tp_total == 14) { /* 470, 3/3/4/4, 5 */
                        priv->magic_not_rop_nr = 0x05;
-                        priv->magicgpc918      = 0x00092493;
                } else
                if (priv->tp_total == 15) { /* 480, 3/4/4/4, 6 */
                        priv->magic_not_rop_nr = 0x06;
-                        priv->magicgpc918      = 0x00088889;
                }
                break;
        case 0xc3: /* 450, 4/0/0/0, 2 */
                priv->magic_not_rop_nr = 0x03;
-                priv->magicgpc918      = 0x00200000;
                break;
        case 0xc4: /* 460, 3/4/0/0, 4 */
                priv->magic_not_rop_nr = 0x01;
-                priv->magicgpc918      = 0x00124925;
+                break;
+        case 0xc1: /* 2/0/0/0, 1 */
+                priv->magic_not_rop_nr = 0x01;
+                break;
+        case 0xc8: /* 4/4/3/4, 5 */
+                priv->magic_not_rop_nr = 0x06;
+                break;
+        case 0xce: /* 4/4/0/0, 4 */
+                priv->magic_not_rop_nr = 0x03;
                break;
        }
@@ -763,13 +872,16 @@ nvc0_graph_create(struct drm_device *dev)
                         priv->tp_nr[3], priv->rop_nr);
                /* use 0xc3's values... */
                priv->magic_not_rop_nr = 0x03;
-                priv->magicgpc918      = 0x00200000;
        }
        NVOBJ_CLASS(dev, 0x902d, GR); /* 2D */
        NVOBJ_CLASS(dev, 0x9039, GR); /* M2MF */
        NVOBJ_MTHD (dev, 0x9039, 0x0500, nvc0_graph_mthd_page_flip);
        NVOBJ_CLASS(dev, 0x9097, GR); /* 3D */
+        if (fermi >= 0x9197)
+                NVOBJ_CLASS(dev, 0x9197, GR); /* 3D (NVC1-) */
+        if (fermi >= 0x9297)
+                NVOBJ_CLASS(dev, 0x9297, GR); /* 3D (NVC8-) */
        NVOBJ_CLASS(dev, 0x90c0, GR); /* COMPUTE */
        return 0;
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.fuc b/drivers/gpu/drm/nouveau/nvc0_graph.fuc
new file mode 100644
index 000000000000..2a4b6dc8f9de
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.fuc
@@ -0,0 +1,400 @@
+/* fuc microcode util functions for nvc0 PGRAPH
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+define(`mmctx_data', `.b32 eval((($2 - 1) << 26) | $1)')
+define(`queue_init', `.skip eval((2 * 4) + ((8 * 4) * 2))')
+ifdef(`include_code', `
+// Error codes
+define(`E_BAD_COMMAND', 0x01)
+define(`E_CMD_OVERFLOW', 0x02)
+// Util macros to help with debugging ucode hangs etc
+define(`T_WAIT', 0)
+define(`T_MMCTX', 1)
+define(`T_STRWAIT', 2)
+define(`T_STRINIT', 3)
+define(`T_AUTO', 4)
+define(`T_CHAN', 5)
+define(`T_LOAD', 6)
+define(`T_SAVE', 7)
+define(`T_LCHAN', 8)
+define(`T_LCTXH', 9)
+define(`trace_set', `
+        mov $r8 0x83c
+        shl b32 $r8 6
+        clear b32 $r9
+        bset $r9 $1
+        iowr I[$r8 + 0x000] $r9         // CC_SCRATCH[7]
+')
+define(`trace_clr', `
+        mov $r8 0x85c
+        shl b32 $r8 6
+        clear b32 $r9
+        bset $r9 $1
+        iowr I[$r8 + 0x000] $r9         // CC_SCRATCH[7]
+')
+// queue_put - add request to queue
+//
+// In : $r13 queue pointer
+//      $r14 command
+//      $r15 data
+//
+queue_put:
+        // make sure we have space..
+        ld b32 $r8 D[$r13 + 0x0]        // GET
+        ld b32 $r9 D[$r13 + 0x4]        // PUT
+        xor $r8 8
+        cmpu b32 $r8 $r9
+        bra ne queue_put_next
+                mov $r15 E_CMD_OVERFLOW
+                call error
+                ret
+        // store cmd/data on queue
+        queue_put_next:
+        and $r8 $r9 7
+        shl b32 $r8 3
+        add b32 $r8 $r13
+        add b32 $r8 8
+        st b32 D[$r8 + 0x0] $r14
+        st b32 D[$r8 + 0x4] $r15
+        // update PUT
+        add b32 $r9 1
+        and $r9 0xf
+        st b32 D[$r13 + 0x4] $r9
+        ret
+// queue_get - fetch request from queue
+//
+// In : $r13 queue pointer
+//
+// Out: $p1  clear on success (data available)
+//      $r14 command
+//      $r15 data
+//
+queue_get:
+        bset $flags $p1
+        ld b32 $r8 D[$r13 + 0x0]        // GET
+        ld b32 $r9 D[$r13 + 0x4]        // PUT
+        cmpu b32 $r8 $r9
+        bra e queue_get_done
+                // fetch first cmd/data pair
+                and $r9 $r8 7
+                shl b32 $r9 3
+                add b32 $r9 $r13
+                add b32 $r9 8
+                ld b32 $r14 D[$r9 + 0x0]
+                ld b32 $r15 D[$r9 + 0x4]
+                // update GET
+                add b32 $r8 1
+                and $r8 0xf
+                st b32 D[$r13 + 0x0] $r8
+                bclr $flags $p1
+queue_get_done:
+        ret
+// nv_rd32 - read 32-bit value from nv register
+//
+// In : $r14 register
+// Out: $r15 value
+//
+nv_rd32:
+        mov $r11 0x728
+        shl b32 $r11 6
+        mov b32 $r12 $r14
+        bset $r12 31                    // MMIO_CTRL_PENDING
+        iowr I[$r11 + 0x000] $r12       // MMIO_CTRL
+        nv_rd32_wait:
+                iord $r12 I[$r11 + 0x000]
+                xbit $r12 $r12 31
+                bra ne nv_rd32_wait
+        mov $r10 6                      // DONE_MMIO_RD
+        call wait_doneo
+        iord $r15 I[$r11 + 0x100]       // MMIO_RDVAL
+        ret
+// nv_wr32 - write 32-bit value to nv register
+//
+// In : $r14 register
+//      $r15 value
+//
+nv_wr32:
+        mov $r11 0x728
+        shl b32 $r11 6
+        iowr I[$r11 + 0x200] $r15       // MMIO_WRVAL
+        mov b32 $r12 $r14
+        bset $r12 31                    // MMIO_CTRL_PENDING
+        bset $r12 30                    // MMIO_CTRL_WRITE
+        iowr I[$r11 + 0x000] $r12       // MMIO_CTRL
+        nv_wr32_wait:
+                iord $r12 I[$r11 + 0x000]
+                xbit $r12 $r12 31
+                bra ne nv_wr32_wait
+        ret
+// (re)set watchdog timer
+//
+// In : $r15 timeout
+//
+watchdog_reset:
+        mov $r8 0x430
+        shl b32 $r8 6
+        bset $r15 31
+        iowr I[$r8 + 0x000] $r15
+        ret
+// clear watchdog timer
+watchdog_clear:
+        mov $r8 0x430
+        shl b32 $r8 6
+        iowr I[$r8 + 0x000] $r0
+        ret
+// wait_done{z,o} - wait on FUC_DONE bit to become clear/set
+//
+// In : $r10 bit to wait on
+//
+define(`wait_done', `
+$1:
+        trace_set(T_WAIT);
+        mov $r8 0x818
+        shl b32 $r8 6
+        iowr I[$r8 + 0x000] $r10        // CC_SCRATCH[6] = wait bit
+        wait_done_$1:
+                mov $r8 0x400
+                shl b32 $r8 6
+                iord $r8 I[$r8 + 0x000] // DONE
+                xbit $r8 $r8 $r10
+                bra $2 wait_done_$1
+        trace_clr(T_WAIT)
+        ret
+')
+wait_done(wait_donez, ne)
+wait_done(wait_doneo, e)
+// mmctx_size - determine size of a mmio list transfer
+//
+// In : $r14 mmio list head
+//      $r15 mmio list tail
+// Out: $r15 transfer size (in bytes)
+//
+mmctx_size:
+        clear b32 $r9
+        nv_mmctx_size_loop:
+                ld b32 $r8 D[$r14]
+                shr b32 $r8 26
+                add b32 $r8 1
+                shl b32 $r8 2
+                add b32 $r9 $r8
+                add b32 $r14 4
+                cmpu b32 $r14 $r15
+                bra ne nv_mmctx_size_loop
+        mov b32 $r15 $r9
+        ret
+// mmctx_xfer - execute a list of mmio transfers
+//
+// In : $r10 flags
+//              bit 0: direction (0 = save, 1 = load)
+//              bit 1: set if first transfer
+//              bit 2: set if last transfer
+//      $r11 base
+//      $r12 mmio list head
+//      $r13 mmio list tail
+//      $r14 multi_stride
+//      $r15 multi_mask
+//
+mmctx_xfer:
+        trace_set(T_MMCTX)
+        mov $r8 0x710
+        shl b32 $r8 6
+        clear b32 $r9
+        or $r11 $r11
+        bra e mmctx_base_disabled
+                iowr I[$r8 + 0x000] $r11        // MMCTX_BASE
+                bset $r9 0                      // BASE_EN
+        mmctx_base_disabled:
+        or $r14 $r14
+        bra e mmctx_multi_disabled
+                iowr I[$r8 + 0x200] $r14        // MMCTX_MULTI_STRIDE
+                iowr I[$r8 + 0x300] $r15        // MMCTX_MULTI_MASK
+                bset $r9 1                      // MULTI_EN
+        mmctx_multi_disabled:
+        add b32 $r8 0x100
+        xbit $r11 $r10 0
+        shl b32 $r11 16                 // DIR
+        bset $r11 12                    // QLIMIT = 0x10
+        xbit $r14 $r10 1
+        shl b32 $r14 17
+        or $r11 $r14                    // START_TRIGGER
+        iowr I[$r8 + 0x000] $r11        // MMCTX_CTRL
+        // loop over the mmio list, and send requests to the hw
+        mmctx_exec_loop:
+                // wait for space in mmctx queue
+                mmctx_wait_free:
+                        iord $r14 I[$r8 + 0x000] // MMCTX_CTRL
+                        and $r14 0x1f
+                        bra e mmctx_wait_free
+                // queue up an entry
+                ld b32 $r14 D[$r12]
+                or $r14 $r9
+                iowr I[$r8 + 0x300] $r14
+                add b32 $r12 4
+                cmpu b32 $r12 $r13
+                bra ne mmctx_exec_loop
+        xbit $r11 $r10 2
+        bra ne mmctx_stop
+                // wait for queue to empty
+                mmctx_fini_wait:
+                        iord $r11 I[$r8 + 0x000]        // MMCTX_CTRL
+                        and $r11 0x1f
+                        cmpu b32 $r11 0x10
+                        bra ne mmctx_fini_wait
+                mov $r10 2                              // DONE_MMCTX
+                call wait_donez
+                bra mmctx_done
+        mmctx_stop:
+                xbit $r11 $r10 0
+                shl b32 $r11 16                 // DIR
+                bset $r11 12                    // QLIMIT = 0x10
+                bset $r11 18                    // STOP_TRIGGER
+                iowr I[$r8 + 0x000] $r11        // MMCTX_CTRL
+                mmctx_stop_wait:
+                        // wait for STOP_TRIGGER to clear
+                        iord $r11 I[$r8 + 0x000] // MMCTX_CTRL
+                        xbit $r11 $r11 18
+                        bra ne mmctx_stop_wait
+        mmctx_done:
+        trace_clr(T_MMCTX)
+        ret
+// Wait for DONE_STRAND
+//
+strand_wait:
+        push $r10
+        mov $r10 2
+        call wait_donez
+        pop $r10
+        ret
+// unknown - call before issuing strand commands
+//
+strand_pre:
+        mov $r8 0x4afc
+        sethi $r8 0x20000
+        mov $r9 0xc
+        iowr I[$r8] $r9
+        call strand_wait
+        ret
+// unknown - call after issuing strand commands
+//
+strand_post:
+        mov $r8 0x4afc
+        sethi $r8 0x20000
+        mov $r9 0xd
+        iowr I[$r8] $r9
+        call strand_wait
+        ret
+// Selects strand set?!
+//
+// In: $r14 id
+//
+strand_set:
+        mov $r10 0x4ffc
+        sethi $r10 0x20000
+        sub b32 $r11 $r10 0x500
+        mov $r12 0xf
+        iowr I[$r10 + 0x000] $r12               // 0x93c = 0xf
+        mov $r12 0xb
+        iowr I[$r11 + 0x000] $r12               // 0x928 = 0xb
+        call strand_wait
+        iowr I[$r10 + 0x000] $r14               // 0x93c = <id>
+        mov $r12 0xa
+        iowr I[$r11 + 0x000] $r12               // 0x928 = 0xa
+        call strand_wait
+        ret
+// Initialise strand context data
+//
+// In : $r15 context base
+// Out: $r15 context size (in bytes)
+//
+// Strandset(?) 3 hardcoded currently
+//
+strand_ctx_init:
+        trace_set(T_STRINIT)
+        call strand_pre
+        mov $r14 3
+        call strand_set
+        mov $r10 0x46fc
+        sethi $r10 0x20000
+        add b32 $r11 $r10 0x400
+        iowr I[$r10 + 0x100] $r0        // STRAND_FIRST_GENE = 0
+        mov $r12 1
+        iowr I[$r11 + 0x000] $r12       // STRAND_CMD = LATCH_FIRST_GENE
+        call strand_wait
+        sub b32 $r12 $r0 1
+        iowr I[$r10 + 0x000] $r12       // STRAND_GENE_CNT = 0xffffffff
+        mov $r12 2
+        iowr I[$r11 + 0x000] $r12       // STRAND_CMD = LATCH_GENE_CNT
+        call strand_wait
+        call strand_post
+        // read the size of each strand, poke the context offset of
+        // each into STRAND_{SAVE,LOAD}_SWBASE now, no need to worry
+        // about it later then.
+        mov $r8 0x880
+        shl b32 $r8 6
+        iord $r9 I[$r8 + 0x000]         // STRANDS
+        add b32 $r8 0x2200
+        shr b32 $r14 $r15 8
+        ctx_init_strand_loop:
+                iowr I[$r8 + 0x000] $r14        // STRAND_SAVE_SWBASE
+                iowr I[$r8 + 0x100] $r14        // STRAND_LOAD_SWBASE
+                iord $r10 I[$r8 + 0x200]        // STRAND_SIZE
+                shr b32 $r10 6
+                add b32 $r10 1
+                add b32 $r14 $r10
+                add b32 $r8 4
+                sub b32 $r9 1
+                bra ne ctx_init_strand_loop
+        shl b32 $r14 8
+        sub b32 $r15 $r14 $r15
+        trace_clr(T_STRINIT)
+        ret
+')
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.h b/drivers/gpu/drm/nouveau/nvc0_graph.h
index f5d184e0689d..55689e997286 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.h
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.h
@@ -57,8 +57,7 @@ struct nvc0_graph_priv {
        struct nouveau_gpuobj *unk4188b4;
        struct nouveau_gpuobj *unk4188b8;
-        u8  magic_not_rop_nr;
+        u8 magic_not_rop_nr;
-        u32 magicgpc918;
 };
 struct nvc0_graph_chan {
@@ -72,4 +71,25 @@ struct nvc0_graph_chan {
 int nvc0_grctx_generate(struct nouveau_channel *);
+/* nvc0_graph.c uses this also to determine supported chipsets */
+static inline u32
+nvc0_graph_class(struct drm_device *dev)
+{
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        switch (dev_priv->chipset) {
+        case 0xc0:
+        case 0xc3:
+        case 0xc4:
+        case 0xce: /* guess, mmio trace shows only 0x9097 state */
+                return 0x9097;
+        case 0xc1:
+                return 0x9197;
+        case 0xc8:
+                return 0x9297;
+        default:
+                return 0;
+        }
+}
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvc0_grctx.c b/drivers/gpu/drm/nouveau/nvc0_grctx.c
index 6df066114133..31018eaf5279 100644
--- a/drivers/gpu/drm/nouveau/nvc0_grctx.c
+++ b/drivers/gpu/drm/nouveau/nvc0_grctx.c
@@ -45,6 +45,9 @@ nv_mthd(struct drm_device *dev, u32 class, u32 mthd, u32 data)
 static void
 nvc0_grctx_generate_9097(struct drm_device *dev)
 {
+        u32 fermi = nvc0_graph_class(dev);
+        u32 mthd;
        nv_mthd(dev, 0x9097, 0x0800, 0x00000000);
        nv_mthd(dev, 0x9097, 0x0840, 0x00000000);
        nv_mthd(dev, 0x9097, 0x0880, 0x00000000);
@@ -824,134 +827,10 @@ nvc0_grctx_generate_9097(struct drm_device *dev)
        nv_mthd(dev, 0x9097, 0x1eb8, 0x00000001);
        nv_mthd(dev, 0x9097, 0x1ed8, 0x00000001);
        nv_mthd(dev, 0x9097, 0x1ef8, 0x00000001);
-        nv_mthd(dev, 0x9097, 0x3400, 0x00000000);
+        if (fermi == 0x9097) {
-        nv_mthd(dev, 0x9097, 0x3404, 0x00000000);
+                for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
-        nv_mthd(dev, 0x9097, 0x3408, 0x00000000);
+                        nv_mthd(dev, 0x9097, mthd, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x340c, 0x00000000);
+        }
-        nv_mthd(dev, 0x9097, 0x3410, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3414, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3418, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x341c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3420, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3424, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3428, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x342c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3430, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3434, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3438, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x343c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3440, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3444, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3448, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x344c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3450, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3454, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3458, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x345c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3460, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3464, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3468, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x346c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3470, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3474, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3478, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x347c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3480, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3484, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3488, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x348c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3490, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3494, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3498, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x349c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34a0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34a4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34a8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34ac, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34b0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34b4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34b8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34bc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34c0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34c4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34c8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34cc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34d0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34d4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34d8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34dc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34e0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34e4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34e8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34ec, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34f0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34f4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34f8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x34fc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3500, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3504, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3508, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x350c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3510, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3514, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3518, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x351c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3520, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3524, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3528, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x352c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3530, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3534, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3538, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x353c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3540, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3544, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3548, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x354c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3550, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3554, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3558, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x355c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3560, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3564, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3568, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x356c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3570, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3574, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3578, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x357c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3580, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3584, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3588, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x358c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3590, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3594, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x3598, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x359c, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35a0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35a4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35a8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35ac, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35b0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35b4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35b8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35bc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35c0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35c4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35c8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35cc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35d0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35d4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35d8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35dc, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35e0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35e4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35e8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35ec, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35f0, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35f4, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35f8, 0x00000000);
-        nv_mthd(dev, 0x9097, 0x35fc, 0x00000000);
        nv_mthd(dev, 0x9097, 0x030c, 0x00000001);
        nv_mthd(dev, 0x9097, 0x1944, 0x00000000);
        nv_mthd(dev, 0x9097, 0x1514, 0x00000000);
@@ -1321,6 +1200,37 @@ nvc0_grctx_generate_9097(struct drm_device *dev)
 }
 static void
+nvc0_grctx_generate_9197(struct drm_device *dev)
+{
+        u32 fermi = nvc0_graph_class(dev);
+        u32 mthd;
+        if (fermi == 0x9197) {
+                for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+                        nv_mthd(dev, 0x9197, mthd, 0x00000000);
+        }
+        nv_mthd(dev, 0x9197, 0x02e4, 0x0000b001);
+}
+static void
+nvc0_grctx_generate_9297(struct drm_device *dev)
+{
+        u32 fermi = nvc0_graph_class(dev);
+        u32 mthd;
+        if (fermi == 0x9297) {
+                for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+                        nv_mthd(dev, 0x9297, mthd, 0x00000000);
+        }
+        nv_mthd(dev, 0x9297, 0x036c, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x0370, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x07a4, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x07a8, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x0374, 0x00000000);
+        nv_mthd(dev, 0x9297, 0x0378, 0x00000020);
+}
+static void
 nvc0_grctx_generate_902d(struct drm_device *dev)
 {
        nv_mthd(dev, 0x902d, 0x0200, 0x000000cf);
@@ -1559,8 +1469,15 @@ nvc0_grctx_generate_unk47xx(struct drm_device *dev)
 static void
 nvc0_grctx_generate_shaders(struct drm_device *dev)
 {
-        nv_wr32(dev, 0x405800, 0x078000bf);
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        nv_wr32(dev, 0x405830, 0x02180000);
+        if (dev_priv->chipset != 0xc1) {
+                nv_wr32(dev, 0x405800, 0x078000bf);
+                nv_wr32(dev, 0x405830, 0x02180000);
+        } else {
+                nv_wr32(dev, 0x405800, 0x0f8000bf);
+                nv_wr32(dev, 0x405830, 0x02180218);
+        }
        nv_wr32(dev, 0x405834, 0x00000000);
        nv_wr32(dev, 0x405838, 0x00000000);
        nv_wr32(dev, 0x405854, 0x00000000);
@@ -1586,10 +1503,16 @@ nvc0_grctx_generate_unk60xx(struct drm_device *dev)
 static void
 nvc0_grctx_generate_unk64xx(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
        nv_wr32(dev, 0x4064a8, 0x00000000);
        nv_wr32(dev, 0x4064ac, 0x00003fff);
        nv_wr32(dev, 0x4064b4, 0x00000000);
        nv_wr32(dev, 0x4064b8, 0x00000000);
+        if (dev_priv->chipset == 0xc1) {
+                nv_wr32(dev, 0x4064c0, 0x80140078);
+                nv_wr32(dev, 0x4064c4, 0x0086ffff);
+        }
 }
 static void
@@ -1622,21 +1545,14 @@ static void
 nvc0_grctx_generate_rop(struct drm_device *dev)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int chipset = dev_priv->chipset;
        /* ROPC_BROADCAST */
        nv_wr32(dev, 0x408800, 0x02802a3c);
        nv_wr32(dev, 0x408804, 0x00000040);
-        nv_wr32(dev, 0x408808, 0x0003e00d);
+        nv_wr32(dev, 0x408808, chipset != 0xc1 ? 0x0003e00d : 0x1003e005);
-        switch (dev_priv->chipset) {
+        nv_wr32(dev, 0x408900, 0x3080b801);
-        case 0xc0:
+        nv_wr32(dev, 0x408904, chipset != 0xc1 ? 0x02000001 : 0x62000001);
-                nv_wr32(dev, 0x408900, 0x0080b801);
-                break;
-        case 0xc3:
-        case 0xc4:
-                nv_wr32(dev, 0x408900, 0x3080b801);
-                break;
-        }
-        nv_wr32(dev, 0x408904, 0x02000001);
        nv_wr32(dev, 0x408908, 0x00c80929);
        nv_wr32(dev, 0x40890c, 0x00000000);
        nv_wr32(dev, 0x408980, 0x0000011d);
@@ -1645,6 +1561,8 @@ nvc0_grctx_generate_rop(struct drm_device *dev)
 static void
 nvc0_grctx_generate_gpc(struct drm_device *dev)
 {
+        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int chipset = dev_priv->chipset;
        int i;
        /* GPC_BROADCAST */
@@ -1676,7 +1594,7 @@ nvc0_grctx_generate_gpc(struct drm_device *dev)
        nv_wr32(dev, 0x41880c, 0x00000000);
        nv_wr32(dev, 0x418810, 0x00000000);
        nv_wr32(dev, 0x418828, 0x00008442);
-        nv_wr32(dev, 0x418830, 0x00000001);
+        nv_wr32(dev, 0x418830, chipset != 0xc1 ? 0x00000001 : 0x10000001);
        nv_wr32(dev, 0x4188d8, 0x00000008);
        nv_wr32(dev, 0x4188e0, 0x01000000);
        nv_wr32(dev, 0x4188e8, 0x00000000);
@@ -1684,7 +1602,7 @@ nvc0_grctx_generate_gpc(struct drm_device *dev)
        nv_wr32(dev, 0x4188f0, 0x00000000);
        nv_wr32(dev, 0x4188f4, 0x00000000);
        nv_wr32(dev, 0x4188f8, 0x00000000);
-        nv_wr32(dev, 0x4188fc, 0x00100000);
+        nv_wr32(dev, 0x4188fc, chipset != 0xc1 ? 0x00100000 : 0x00100018);
        nv_wr32(dev, 0x41891c, 0x00ff00ff);
        nv_wr32(dev, 0x418924, 0x00000000);
        nv_wr32(dev, 0x418928, 0x00ffff00);
@@ -1715,6 +1633,8 @@ nvc0_grctx_generate_gpc(struct drm_device *dev)
        nv_wr32(dev, 0x418c24, 0x00000000);
        nv_wr32(dev, 0x418c28, 0x00000000);
        nv_wr32(dev, 0x418c2c, 0x00000000);
+        if (chipset == 0xc1)
+                nv_wr32(dev, 0x418c6c, 0x00000001);
        nv_wr32(dev, 0x418c80, 0x20200004);
        nv_wr32(dev, 0x418c8c, 0x00000001);
        nv_wr32(dev, 0x419000, 0x00000780);
@@ -1727,10 +1647,13 @@ static void
 nvc0_grctx_generate_tp(struct drm_device *dev)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        int chipset = dev_priv->chipset;
        /* GPC_BROADCAST.TP_BROADCAST */
+        nv_wr32(dev, 0x419818, 0x00000000);
+        nv_wr32(dev, 0x41983c, 0x00038bc7);
        nv_wr32(dev, 0x419848, 0x00000000);
-        nv_wr32(dev, 0x419864, 0x0000012a);
+        nv_wr32(dev, 0x419864, chipset != 0xc1 ? 0x0000012a : 0x00000129);
        nv_wr32(dev, 0x419888, 0x00000000);
        nv_wr32(dev, 0x419a00, 0x000001f0);
        nv_wr32(dev, 0x419a04, 0x00000001);
@@ -1740,8 +1663,8 @@ nvc0_grctx_generate_tp(struct drm_device *dev)
        nv_wr32(dev, 0x419a14, 0x00000200);
        nv_wr32(dev, 0x419a1c, 0x00000000);
        nv_wr32(dev, 0x419a20, 0x00000800);
-        if (dev_priv->chipset != 0xc0)
+        if (chipset != 0xc0 && chipset != 0xc8)
-                nv_wr32(dev, 0x00419ac4, 0x0007f440); /* 0xc3 */
+                nv_wr32(dev, 0x00419ac4, 0x0007f440);
        nv_wr32(dev, 0x419b00, 0x0a418820);
        nv_wr32(dev, 0x419b04, 0x062080e6);
        nv_wr32(dev, 0x419b08, 0x020398a4);
@@ -1749,17 +1672,19 @@ nvc0_grctx_generate_tp(struct drm_device *dev)
        nv_wr32(dev, 0x419b10, 0x0a418820);
        nv_wr32(dev, 0x419b14, 0x000000e6);
        nv_wr32(dev, 0x419bd0, 0x00900103);
-        nv_wr32(dev, 0x419be0, 0x00000001);
+        nv_wr32(dev, 0x419be0, chipset != 0xc1 ? 0x00000001 : 0x00400001);
        nv_wr32(dev, 0x419be4, 0x00000000);
        nv_wr32(dev, 0x419c00, 0x00000002);
        nv_wr32(dev, 0x419c04, 0x00000006);
        nv_wr32(dev, 0x419c08, 0x00000002);
        nv_wr32(dev, 0x419c20, 0x00000000);
-        nv_wr32(dev, 0x419cbc, 0x28137606);
+        nv_wr32(dev, 0x419cb0, 0x00060048); //XXX: 0xce 0x00020048
        nv_wr32(dev, 0x419ce8, 0x00000000);
        nv_wr32(dev, 0x419cf4, 0x00000183);
-        nv_wr32(dev, 0x419d20, 0x02180000);
+        nv_wr32(dev, 0x419d20, chipset != 0xc1 ? 0x02180000 : 0x12180000);
        nv_wr32(dev, 0x419d24, 0x00001fff);
+        if (chipset == 0xc1)
+                nv_wr32(dev, 0x419d44, 0x02180218);
        nv_wr32(dev, 0x419e04, 0x00000000);
        nv_wr32(dev, 0x419e08, 0x00000000);
        nv_wr32(dev, 0x419e0c, 0x00000000);
@@ -1785,11 +1710,11 @@ nvc0_grctx_generate_tp(struct drm_device *dev)
        nv_wr32(dev, 0x419e8c, 0x00000000);
        nv_wr32(dev, 0x419e90, 0x00000000);
        nv_wr32(dev, 0x419e98, 0x00000000);
-        if (dev_priv->chipset != 0xc0)
+        if (chipset != 0xc0 && chipset != 0xc8)
                nv_wr32(dev, 0x419ee0, 0x00011110);
        nv_wr32(dev, 0x419f50, 0x00000000);
        nv_wr32(dev, 0x419f54, 0x00000000);
-        if (dev_priv->chipset != 0xc0)
+        if (chipset != 0xc0 && chipset != 0xc8)
                nv_wr32(dev, 0x419f58, 0x00000000);
 }
@@ -1801,6 +1726,7 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
        struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
        struct drm_device *dev = chan->dev;
        int i, gpc, tp, id;
+        u32 fermi = nvc0_graph_class(dev);
        u32 r000260, tmp;
        r000260 = nv_rd32(dev, 0x000260);
@@ -1857,10 +1783,11 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
        nv_wr32(dev, 0x40587c, 0x00000000);
        if (1) {
-                const u8 chipset_tp_max[] = { 16, 0, 0, 4, 8 };
+                const u8 chipset_tp_max[] = { 16, 4, 0, 4, 8, 0, 0, 0,
+                                              16, 0, 0, 0, 0, 0, 8, 0 };
                u8 max = chipset_tp_max[dev_priv->chipset & 0x0f];
                u8 tpnr[GPC_MAX];
-                u8 data[32];
+                u8 data[TP_MAX];
                memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
                memset(data, 0x1f, sizeof(data));
@@ -2633,6 +2560,8 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
        nv_icmd(dev, 0x0000053f, 0xffff0000);
        nv_icmd(dev, 0x00000585, 0x0000003f);
        nv_icmd(dev, 0x00000576, 0x00000003);
+        if (dev_priv->chipset == 0xc1)
+                nv_icmd(dev, 0x0000057b, 0x00000059);
        nv_icmd(dev, 0x00000586, 0x00000040);
        nv_icmd(dev, 0x00000582, 0x00000080);
        nv_icmd(dev, 0x00000583, 0x00000080);
@@ -2865,6 +2794,10 @@ nvc0_grctx_generate(struct nouveau_channel *chan)
        nv_wr32(dev, 0x404154, 0x00000400);
        nvc0_grctx_generate_9097(dev);
+        if (fermi >= 0x9197)
+                nvc0_grctx_generate_9197(dev);
+        if (fermi >= 0x9297)
+                nvc0_grctx_generate_9297(dev);
        nvc0_grctx_generate_902d(dev);
        nvc0_grctx_generate_9039(dev);
        nvc0_grctx_generate_90c0(dev);
diff --git a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc
new file mode 100644
index 000000000000..0ec2add72a76
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc
@@ -0,0 +1,474 @@
+/* fuc microcode for nvc0 PGRAPH/GPC
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+/* To build:
+ *    m4 nvc0_grgpc.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grgpc.fuc.h
+ */
+/* TODO
+ * - bracket certain functions with scratch writes, useful for debugging
+ * - watchdog timer around ctx operations
+ */
+.section nvc0_grgpc_data
+include(`nvc0_graph.fuc')
+gpc_id:                 .b32 0
+gpc_mmio_list_head:     .b32 0
+gpc_mmio_list_tail:     .b32 0
+tpc_count:              .b32 0
+tpc_mask:               .b32 0
+tpc_mmio_list_head:     .b32 0
+tpc_mmio_list_tail:     .b32 0
+cmd_queue:              queue_init
+// chipset descriptions
+chipsets:
+.b8  0xc0 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc0_tpc_mmio_tail
+.b8  0xc1 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc1_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc1_tpc_mmio_tail
+.b8  0xc3 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc3_tpc_mmio_tail
+.b8  0xc4 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc3_tpc_mmio_tail
+.b8  0xc8 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc0_tpc_mmio_tail
+.b8  0xce 0 0 0
+.b16 nvc0_gpc_mmio_head
+.b16 nvc0_gpc_mmio_tail
+.b16 nvc0_tpc_mmio_head
+.b16 nvc3_tpc_mmio_tail
+.b8  0 0 0 0
+// GPC mmio lists
+nvc0_gpc_mmio_head:
+mmctx_data(0x000380, 1)
+mmctx_data(0x000400, 6)
+mmctx_data(0x000450, 9)
+mmctx_data(0x000600, 1)
+mmctx_data(0x000684, 1)
+mmctx_data(0x000700, 5)
+mmctx_data(0x000800, 1)
+mmctx_data(0x000808, 3)
+mmctx_data(0x000828, 1)
+mmctx_data(0x000830, 1)
+mmctx_data(0x0008d8, 1)
+mmctx_data(0x0008e0, 1)
+mmctx_data(0x0008e8, 6)
+mmctx_data(0x00091c, 1)
+mmctx_data(0x000924, 3)
+mmctx_data(0x000b00, 1)
+mmctx_data(0x000b08, 6)
+mmctx_data(0x000bb8, 1)
+mmctx_data(0x000c08, 1)
+mmctx_data(0x000c10, 8)
+mmctx_data(0x000c80, 1)
+mmctx_data(0x000c8c, 1)
+mmctx_data(0x001000, 3)
+mmctx_data(0x001014, 1)
+nvc0_gpc_mmio_tail:
+mmctx_data(0x000c6c, 1);
+nvc1_gpc_mmio_tail:
+// TPC mmio lists
+nvc0_tpc_mmio_head:
+mmctx_data(0x000018, 1)
+mmctx_data(0x00003c, 1)
+mmctx_data(0x000048, 1)
+mmctx_data(0x000064, 1)
+mmctx_data(0x000088, 1)
+mmctx_data(0x000200, 6)
+mmctx_data(0x00021c, 2)
+mmctx_data(0x000300, 6)
+mmctx_data(0x0003d0, 1)
+mmctx_data(0x0003e0, 2)
+mmctx_data(0x000400, 3)
+mmctx_data(0x000420, 1)
+mmctx_data(0x0004b0, 1)
+mmctx_data(0x0004e8, 1)
+mmctx_data(0x0004f4, 1)
+mmctx_data(0x000520, 2)
+mmctx_data(0x000604, 4)
+mmctx_data(0x000644, 20)
+mmctx_data(0x000698, 1)
+mmctx_data(0x000750, 2)
+nvc0_tpc_mmio_tail:
+mmctx_data(0x000758, 1)
+mmctx_data(0x0002c4, 1)
+mmctx_data(0x0004bc, 1)
+mmctx_data(0x0006e0, 1)
+nvc3_tpc_mmio_tail:
+mmctx_data(0x000544, 1)
+nvc1_tpc_mmio_tail:
+.section nvc0_grgpc_code
+bra init
+define(`include_code')
+include(`nvc0_graph.fuc')
+// reports an exception to the host
+//
+// In: $r15 error code (see nvc0_graph.fuc)
+//
+error:
+        push $r14
+        mov $r14 -0x67ec        // 0x9814
+        sethi $r14 0x400000
+        call nv_wr32            // HUB_CTXCTL_CC_SCRATCH[5] = error code
+        add b32 $r14 0x41c
+        mov $r15 1
+        call nv_wr32            // HUB_CTXCTL_INTR_UP_SET
+        pop $r14
+        ret
+// GPC fuc initialisation, executed by triggering ucode start, will
+// fall through to main loop after completion.
+//
+// Input:
+//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
+//   CC_SCRATCH[1]: context base
+//
+// Output:
+//   CC_SCRATCH[0]:
+//           31:31: set to signal completion
+//   CC_SCRATCH[1]:
+//            31:0: GPC context size
+//
+init:
+        clear b32 $r0
+        mov $sp $r0
+        // enable fifo access
+        mov $r1 0x1200
+        mov $r2 2
+        iowr I[$r1 + 0x000] $r2         // FIFO_ENABLE
+        // setup i0 handler, and route all interrupts to it
+        mov $r1 ih
+        mov $iv0 $r1
+        mov $r1 0x400
+        iowr I[$r1 + 0x300] $r0         // INTR_DISPATCH
+        // enable fifo interrupt
+        mov $r2 4
+        iowr I[$r1 + 0x000] $r2         // INTR_EN_SET
+        // enable interrupts
+        bset $flags ie0
+        // figure out which GPC we are, and how many TPCs we have
+        mov $r1 0x608
+        shl b32 $r1 6
+        iord $r2 I[$r1 + 0x000]         // UNITS
+        mov $r3 1
+        and $r2 0x1f
+        shl b32 $r3 $r2
+        sub b32 $r3 1
+        st b32 D[$r0 + tpc_count] $r2
+        st b32 D[$r0 + tpc_mask] $r3
+        add b32 $r1 0x400
+        iord $r2 I[$r1 + 0x000]         // MYINDEX
+        st b32 D[$r0 + gpc_id] $r2
+        // find context data for this chipset
+        mov $r2 0x800
+        shl b32 $r2 6
+        iord $r2 I[$r2 + 0x000]         // CC_SCRATCH[0]
+        mov $r1 chipsets - 12
+        init_find_chipset:
+                add b32 $r1 12
+                ld b32 $r3 D[$r1 + 0x00]
+                cmpu b32 $r3 $r2
+                bra e init_context
+                cmpu b32 $r3 0
+                bra ne init_find_chipset
+                // unknown chipset
+                ret
+        // initialise context base, and size tracking
+        init_context:
+        mov $r2 0x800
+        shl b32 $r2 6
+        iord $r2 I[$r2 + 0x100] // CC_SCRATCH[1], initial base
+        clear b32 $r3           // track GPC context size here
+        // set mmctx base addresses now so we don't have to do it later,
+        // they don't currently ever change
+        mov $r4 0x700
+        shl b32 $r4 6
+        shr b32 $r5 $r2 8
+        iowr I[$r4 + 0x000] $r5         // MMCTX_SAVE_SWBASE
+        iowr I[$r4 + 0x100] $r5         // MMCTX_LOAD_SWBASE
+        // calculate GPC mmio context size, store the chipset-specific
+        // mmio list pointers somewhere we can get at them later without
+        // re-parsing the chipset list
+        clear b32 $r14
+        clear b32 $r15
+        ld b16 $r14 D[$r1 + 4]
+        ld b16 $r15 D[$r1 + 6]
+        st b16 D[$r0 + gpc_mmio_list_head] $r14
+        st b16 D[$r0 + gpc_mmio_list_tail] $r15
+        call mmctx_size
+        add b32 $r2 $r15
+        add b32 $r3 $r15
+        // calculate per-TPC mmio context size, store the list pointers
+        ld b16 $r14 D[$r1 + 8]
+        ld b16 $r15 D[$r1 + 10]
+        st b16 D[$r0 + tpc_mmio_list_head] $r14
+        st b16 D[$r0 + tpc_mmio_list_tail] $r15
+        call mmctx_size
+        ld b32 $r14 D[$r0 + tpc_count]
+        mulu $r14 $r15
+        add b32 $r2 $r14
+        add b32 $r3 $r14
+        // round up base/size to 256 byte boundary (for strand SWBASE)
+        add b32 $r4 0x1300
+        shr b32 $r3 2
+        iowr I[$r4 + 0x000] $r3         // MMCTX_LOAD_COUNT, wtf for?!?
+        shr b32 $r2 8
+        shr b32 $r3 6
+        add b32 $r2 1
+        add b32 $r3 1
+        shl b32 $r2 8
+        shl b32 $r3 8
+        // calculate size of strand context data
+        mov b32 $r15 $r2
+        call strand_ctx_init
+        add b32 $r3 $r15
+        // save context size, and tell HUB we're done
+        mov $r1 0x800
+        shl b32 $r1 6
+        iowr I[$r1 + 0x100] $r3         // CC_SCRATCH[1]  = context size
+        add b32 $r1 0x800
+        clear b32 $r2
+        bset $r2 31
+        iowr I[$r1 + 0x000] $r2         // CC_SCRATCH[0] |= 0x80000000
+// Main program loop, very simple, sleeps until woken up by the interrupt
+// handler, pulls a command from the queue and executes its handler
+//
+main:
+        bset $flags $p0
+        sleep $p0
+        mov $r13 cmd_queue
+        call queue_get
+        bra $p1 main
+        // 0x0000-0x0003 are all context transfers
+        cmpu b32 $r14 0x04
+        bra nc main_not_ctx_xfer
+                // fetch $flags and mask off $p1/$p2
+                mov $r1 $flags
+                mov $r2 0x0006
+                not b32 $r2
+                and $r1 $r2
+                // set $p1/$p2 according to transfer type
+                shl b32 $r14 1
+                or $r1 $r14
+                mov $flags $r1
+                // transfer context data
+                call ctx_xfer
+                bra main
+        main_not_ctx_xfer:
+        shl b32 $r15 $r14 16
+        or $r15 E_BAD_COMMAND
+        call error
+        bra main
+// interrupt handler
+ih:
+        push $r8
+        mov $r8 $flags
+        push $r8
+        push $r9
+        push $r10
+        push $r11
+        push $r13
+        push $r14
+        push $r15
+        // incoming fifo command?
+        iord $r10 I[$r0 + 0x200]        // INTR
+        and $r11 $r10 0x00000004
+        bra e ih_no_fifo
+                // queue incoming fifo command for later processing
+                mov $r11 0x1900
+                mov $r13 cmd_queue
+                iord $r14 I[$r11 + 0x100]       // FIFO_CMD
+                iord $r15 I[$r11 + 0x000]       // FIFO_DATA
+                call queue_put
+                add b32 $r11 0x400
+                mov $r14 1
+                iowr I[$r11 + 0x000] $r14       // FIFO_ACK
+        // ack, and wake up main()
+        ih_no_fifo:
+        iowr I[$r0 + 0x100] $r10        // INTR_ACK
+        pop $r15
+        pop $r14
+        pop $r13
+        pop $r11
+        pop $r10
+        pop $r9
+        pop $r8
+        mov $flags $r8
+        pop $r8
+        bclr $flags $p0
+        iret
+// Set this GPC's bit in HUB_BAR, used to signal completion of various
+// activities to the HUB fuc
+//
+hub_barrier_done:
+        mov $r15 1
+        ld b32 $r14 D[$r0 + gpc_id]
+        shl b32 $r15 $r14
+        mov $r14 -0x6be8        // 0x409418 - HUB_BAR_SET
+        sethi $r14 0x400000
+        call nv_wr32
+        ret
+// Disables various things, waits a bit, and re-enables them..
+//
+// Not sure how exactly this helps, perhaps "ENABLE" is not such a
+// good description for the bits we turn off?  Anyways, without this,
+// funny things happen.
+//
+ctx_redswitch:
+        mov $r14 0x614
+        shl b32 $r14 6
+        mov $r15 0x020
+        iowr I[$r14] $r15       // GPC_RED_SWITCH = POWER
+        mov $r15 8
+        ctx_redswitch_delay:
+                sub b32 $r15 1
+                bra ne ctx_redswitch_delay
+        mov $r15 0xa20
+        iowr I[$r14] $r15       // GPC_RED_SWITCH = UNK11, ENABLE, POWER
+        ret
+// Transfer GPC context data between GPU and storage area
+//
+// In: $r15 context base address
+//     $p1 clear on save, set on load
+//     $p2 set if opposite direction done/will be done, so:
+//              on save it means: "a load will follow this save"
+//              on load it means: "a save preceeded this load"
+//
+ctx_xfer:
+        // set context base address
+        mov $r1 0xa04
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r15// MEM_BASE
+        bra not $p1 ctx_xfer_not_load
+                call ctx_redswitch
+        ctx_xfer_not_load:
+        // strands
+        mov $r1 0x4afc
+        sethi $r1 0x20000
+        mov $r2 0xc
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x0c
+        call strand_wait
+        mov $r2 0x47fc
+        sethi $r2 0x20000
+        iowr I[$r2] $r0         // STRAND_FIRST_GENE(0x3f) = 0x00
+        xbit $r2 $flags $p1
+        add b32 $r2 3
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)
+        // mmio context
+        xbit $r10 $flags $p1    // direction
+        or $r10 2               // first
+        mov $r11 0x0000
+        sethi $r11 0x500000
+        ld b32 $r12 D[$r0 + gpc_id]
+        shl b32 $r12 15
+        add b32 $r11 $r12       // base = NV_PGRAPH_GPCn
+        ld b32 $r12 D[$r0 + gpc_mmio_list_head]
+        ld b32 $r13 D[$r0 + gpc_mmio_list_tail]
+        mov $r14 0              // not multi
+        call mmctx_xfer
+        // per-TPC mmio context
+        xbit $r10 $flags $p1    // direction
+        or $r10 4               // last
+        mov $r11 0x4000
+        sethi $r11 0x500000     // base = NV_PGRAPH_GPC0_TPC0
+        ld b32 $r12 D[$r0 + gpc_id]
+        shl b32 $r12 15
+        add b32 $r11 $r12       // base = NV_PGRAPH_GPCn_TPC0
+        ld b32 $r12 D[$r0 + tpc_mmio_list_head]
+        ld b32 $r13 D[$r0 + tpc_mmio_list_tail]
+        ld b32 $r15 D[$r0 + tpc_mask]
+        mov $r14 0x800          // stride = 0x800
+        call mmctx_xfer
+        // wait for strands to finish
+        call strand_wait
+        // if load, or a save without a load following, do some
+        // unknown stuff that's done after finishing a block of
+        // strand commands
+        bra $p1 ctx_xfer_post
+        bra not $p2 ctx_xfer_done
+        ctx_xfer_post:
+                mov $r1 0x4afc
+                sethi $r1 0x20000
+                mov $r2 0xd
+                iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x0d
+                call strand_wait
+        // mark completion in HUB's barrier
+        ctx_xfer_done:
+        call hub_barrier_done
+        ret
+.align 256
diff --git a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h
new file mode 100644
index 000000000000..1896c898f5ba
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h
@@ -0,0 +1,483 @@
+uint32_t nvc0_grgpc_data[] = {
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x000000c0,
+        0x011000b0,
+        0x01640114,
+        0x000000c1,
+        0x011400b0,
+        0x01780114,
+        0x000000c3,
+        0x011000b0,
+        0x01740114,
+        0x000000c4,
+        0x011000b0,
+        0x01740114,
+        0x000000c8,
+        0x011000b0,
+        0x01640114,
+        0x000000ce,
+        0x011000b0,
+        0x01740114,
+        0x00000000,
+        0x00000380,
+        0x14000400,
+        0x20000450,
+        0x00000600,
+        0x00000684,
+        0x10000700,
+        0x00000800,
+        0x08000808,
+        0x00000828,
+        0x00000830,
+        0x000008d8,
+        0x000008e0,
+        0x140008e8,
+        0x0000091c,
+        0x08000924,
+        0x00000b00,
+        0x14000b08,
+        0x00000bb8,
+        0x00000c08,
+        0x1c000c10,
+        0x00000c80,
+        0x00000c8c,
+        0x08001000,
+        0x00001014,
+        0x00000c6c,
+        0x00000018,
+        0x0000003c,
+        0x00000048,
+        0x00000064,
+        0x00000088,
+        0x14000200,
+        0x0400021c,
+        0x14000300,
+        0x000003d0,
+        0x040003e0,
+        0x08000400,
+        0x00000420,
+        0x000004b0,
+        0x000004e8,
+        0x000004f4,
+        0x04000520,
+        0x0c000604,
+        0x4c000644,
+        0x00000698,
+        0x04000750,
+        0x00000758,
+        0x000002c4,
+        0x000004bc,
+        0x000006e0,
+        0x00000544,
+};
+uint32_t nvc0_grgpc_code[] = {
+        0x03060ef5,
+        0x9800d898,
+        0x86f001d9,
+        0x0489b808,
+        0xf00c1bf4,
+        0x21f502f7,
+        0x00f802ec,
+        0xb60798c4,
+        0x8dbb0384,
+        0x0880b600,
+        0x80008e80,
+        0x90b6018f,
+        0x0f94f001,
+        0xf801d980,
+        0x0131f400,
+        0x9800d898,
+        0x89b801d9,
+        0x210bf404,
+        0xb60789c4,
+        0x9dbb0394,
+        0x0890b600,
+        0x98009e98,
+        0x80b6019f,
+        0x0f84f001,
+        0xf400d880,
+        0x00f80132,
+        0x0728b7f1,
+        0xb906b4b6,
+        0xc9f002ec,
+        0x00bcd01f,
+        0xc800bccf,
+        0x1bf41fcc,
+        0x06a7f0fa,
+        0x010321f5,
+        0xf840bfcf,
+        0x28b7f100,
+        0x06b4b607,
+        0xb980bfd0,
+        0xc9f002ec,
+        0x1ec9f01f,
+        0xcf00bcd0,
+        0xccc800bc,
+        0xfa1bf41f,
+        0x87f100f8,
+        0x84b60430,
+        0x1ff9f006,
+        0xf8008fd0,
+        0x3087f100,
+        0x0684b604,
+        0xf80080d0,
+        0x3c87f100,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d000,
+        0x081887f1,
+        0xd00684b6,
+        0x87f1008a,
+        0x84b60400,
+        0x0088cf06,
+        0xf4888aff,
+        0x87f1f31b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00099,
+        0xf100f800,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00099f0,
+        0x87f10089,
+        0x84b60818,
+        0x008ad006,
+        0x040087f1,
+        0xcf0684b6,
+        0x8aff0088,
+        0xf30bf488,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0099f094,
+        0xf80089d0,
+        0x9894bd00,
+        0x85b600e8,
+        0x0180b61a,
+        0xbb0284b6,
+        0xe0b60098,
+        0x04efb804,
+        0xb9eb1bf4,
+        0x00f8029f,
+        0x083c87f1,
+        0xbd0684b6,
+        0x0199f094,
+        0xf10089d0,
+        0xb6071087,
+        0x94bd0684,
+        0xf405bbfd,
+        0x8bd0090b,
+        0x0099f000,
+        0xf405eefd,
+        0x8ed00c0b,
+        0xc08fd080,
+        0xb70199f0,
+        0xc8010080,
+        0xb4b600ab,
+        0x0cb9f010,
+        0xb601aec8,
+        0xbefd11e4,
+        0x008bd005,
+        0xf0008ecf,
+        0x0bf41fe4,
+        0x00ce98fa,
+        0xd005e9fd,
+        0xc0b6c08e,
+        0x04cdb804,
+        0xc8e81bf4,
+        0x1bf402ab,
+        0x008bcf18,
+        0xb01fb4f0,
+        0x1bf410b4,
+        0x02a7f0f7,
+        0xf4c921f4,
+        0xabc81b0e,
+        0x10b4b600,
+        0xf00cb9f0,
+        0x8bd012b9,
+        0x008bcf00,
+        0xf412bbc8,
+        0x87f1fa1b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00199,
+        0xf900f800,
+        0x02a7f0a0,
+        0xfcc921f4,
+        0xf100f8a0,
+        0xf04afc87,
+        0x97f00283,
+        0x0089d00c,
+        0x020721f5,
+        0x87f100f8,
+        0x83f04afc,
+        0x0d97f002,
+        0xf50089d0,
+        0xf8020721,
+        0xfca7f100,
+        0x02a3f04f,
+        0x0500aba2,
+        0xd00fc7f0,
+        0xc7f000ac,
+        0x00bcd00b,
+        0x020721f5,
+        0xf000aed0,
+        0xbcd00ac7,
+        0x0721f500,
+        0xf100f802,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x21f50089,
+        0xe7f00213,
+        0x3921f503,
+        0xfca7f102,
+        0x02a3f046,
+        0x0400aba0,
+        0xf040a0d0,
+        0xbcd001c7,
+        0x0721f500,
+        0x010c9202,
+        0xf000acd0,
+        0xbcd002c7,
+        0x0721f500,
+        0x2621f502,
+        0x8087f102,
+        0x0684b608,
+        0xb70089cf,
+        0x95220080,
+        0x8ed008fe,
+        0x408ed000,
+        0xb6808acf,
+        0xa0b606a5,
+        0x00eabb01,
+        0xb60480b6,
+        0x1bf40192,
+        0x08e4b6e8,
+        0xf1f2efbc,
+        0xb6085c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x00f80089,
+        0xe7f1e0f9,
+        0xe3f09814,
+        0x8d21f440,
+        0x041ce0b7,
+        0xf401f7f0,
+        0xe0fc8d21,
+        0x04bd00f8,
+        0xf10004fe,
+        0xf0120017,
+        0x12d00227,
+        0x3e17f100,
+        0x0010fe04,
+        0x040017f1,
+        0xf0c010d0,
+        0x12d00427,
+        0x1031f400,
+        0x060817f1,
+        0xcf0614b6,
+        0x37f00012,
+        0x1f24f001,
+        0xb60432bb,
+        0x02800132,
+        0x04038003,
+        0x040010b7,
+        0x800012cf,
+        0x27f10002,
+        0x24b60800,
+        0x0022cf06,
+        0xb65817f0,
+        0x13980c10,
+        0x0432b800,
+        0xb00b0bf4,
+        0x1bf40034,
+        0xf100f8f1,
+        0xb6080027,
+        0x22cf0624,
+        0xf134bd40,
+        0xb6070047,
+        0x25950644,
+        0x0045d008,
+        0xbd4045d0,
+        0x58f4bde4,
+        0x1f58021e,
+        0x020e4003,
+        0xf5040f40,
+        0xbb013d21,
+        0x3fbb002f,
+        0x041e5800,
+        0x40051f58,
+        0x0f400a0e,
+        0x3d21f50c,
+        0x030e9801,
+        0xbb00effd,
+        0x3ebb002e,
+        0x0040b700,
+        0x0235b613,
+        0xb60043d0,
+        0x35b60825,
+        0x0120b606,
+        0xb60130b6,
+        0x34b60824,
+        0x022fb908,
+        0x026321f5,
+        0xf1003fbb,
+        0xb6080017,
+        0x13d00614,
+        0x0010b740,
+        0xf024bd08,
+        0x12d01f29,
+        0x0031f400,
+        0xf00028f4,
+        0x21f41cd7,
+        0xf401f439,
+        0xf404e4b0,
+        0x81fe1e18,
+        0x0627f001,
+        0x12fd20bd,
+        0x01e4b604,
+        0xfe051efd,
+        0x21f50018,
+        0x0ef404c3,
+        0x10ef94d3,
+        0xf501f5f0,
+        0xf402ec21,
+        0x80f9c60e,
+        0xf90188fe,
+        0xf990f980,
+        0xf9b0f9a0,
+        0xf9e0f9d0,
+        0x800acff0,
+        0xf404abc4,
+        0xb7f11d0b,
+        0xd7f01900,
+        0x40becf1c,
+        0xf400bfcf,
+        0xb0b70421,
+        0xe7f00400,
+        0x00bed001,
+        0xfc400ad0,
+        0xfce0fcf0,
+        0xfcb0fcd0,
+        0xfc90fca0,
+        0x0088fe80,
+        0x32f480fc,
+        0xf001f800,
+        0x0e9801f7,
+        0x04febb00,
+        0x9418e7f1,
+        0xf440e3f0,
+        0x00f88d21,
+        0x0614e7f1,
+        0xf006e4b6,
+        0xefd020f7,
+        0x08f7f000,
+        0xf401f2b6,
+        0xf7f1fd1b,
+        0xefd00a20,
+        0xf100f800,
+        0xb60a0417,
+        0x1fd00614,
+        0x0711f400,
+        0x04a421f5,
+        0x4afc17f1,
+        0xf00213f0,
+        0x12d00c27,
+        0x0721f500,
+        0xfc27f102,
+        0x0223f047,
+        0xf00020d0,
+        0x20b6012c,
+        0x0012d003,
+        0xf001acf0,
+        0xb7f002a5,
+        0x50b3f000,
+        0xb6000c98,
+        0xbcbb0fc4,
+        0x010c9800,
+        0xf0020d98,
+        0x21f500e7,
+        0xacf0015c,
+        0x04a5f001,
+        0x4000b7f1,
+        0x9850b3f0,
+        0xc4b6000c,
+        0x00bcbb0f,
+        0x98050c98,
+        0x0f98060d,
+        0x00e7f104,
+        0x5c21f508,
+        0x0721f501,
+        0x0601f402,
+        0xf11412f4,
+        0xf04afc17,
+        0x27f00213,
+        0x0012d00d,
+        0x020721f5,
+        0x048f21f5,
+        0x000000f8,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc
new file mode 100644
index 000000000000..a1a599124cf4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc
@@ -0,0 +1,808 @@
+/* fuc microcode for nvc0 PGRAPH/HUB
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+/* To build:
+ *    m4 nvc0_grhub.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grhub.fuc.h
+ */
+.section nvc0_grhub_data
+include(`nvc0_graph.fuc')
+gpc_count:              .b32 0
+rop_count:              .b32 0
+cmd_queue:              queue_init
+hub_mmio_list_head:     .b32 0
+hub_mmio_list_tail:     .b32 0
+ctx_current:            .b32 0
+chipsets:
+.b8  0xc0 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xc1 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc1_hub_mmio_tail
+.b8  0xc3 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xc4 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xc8 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0xce 0 0 0
+.b16 nvc0_hub_mmio_head
+.b16 nvc0_hub_mmio_tail
+.b8  0 0 0 0
+nvc0_hub_mmio_head:
+mmctx_data(0x17e91c, 2)
+mmctx_data(0x400204, 2)
+mmctx_data(0x404004, 11)
+mmctx_data(0x404044, 1)
+mmctx_data(0x404094, 14)
+mmctx_data(0x4040d0, 7)
+mmctx_data(0x4040f8, 1)
+mmctx_data(0x404130, 3)
+mmctx_data(0x404150, 3)
+mmctx_data(0x404164, 2)
+mmctx_data(0x404174, 3)
+mmctx_data(0x404200, 8)
+mmctx_data(0x404404, 14)
+mmctx_data(0x404460, 4)
+mmctx_data(0x404480, 1)
+mmctx_data(0x404498, 1)
+mmctx_data(0x404604, 4)
+mmctx_data(0x404618, 32)
+mmctx_data(0x404698, 21)
+mmctx_data(0x4046f0, 2)
+mmctx_data(0x404700, 22)
+mmctx_data(0x405800, 1)
+mmctx_data(0x405830, 3)
+mmctx_data(0x405854, 1)
+mmctx_data(0x405870, 4)
+mmctx_data(0x405a00, 2)
+mmctx_data(0x405a18, 1)
+mmctx_data(0x406020, 1)
+mmctx_data(0x406028, 4)
+mmctx_data(0x4064a8, 2)
+mmctx_data(0x4064b4, 2)
+mmctx_data(0x407804, 1)
+mmctx_data(0x40780c, 6)
+mmctx_data(0x4078bc, 1)
+mmctx_data(0x408000, 7)
+mmctx_data(0x408064, 1)
+mmctx_data(0x408800, 3)
+mmctx_data(0x408900, 4)
+mmctx_data(0x408980, 1)
+nvc0_hub_mmio_tail:
+mmctx_data(0x4064c0, 2)
+nvc1_hub_mmio_tail:
+.align 256
+chan_data:
+chan_mmio_count:        .b32 0
+chan_mmio_address:      .b32 0
+.align 256
+xfer_data:              .b32 0
+.section nvc0_grhub_code
+bra init
+define(`include_code')
+include(`nvc0_graph.fuc')
+// reports an exception to the host
+//
+// In: $r15 error code (see nvc0_graph.fuc)
+//
+error:
+        push $r14
+        mov $r14 0x814
+        shl b32 $r14 6
+        iowr I[$r14 + 0x000] $r15       // CC_SCRATCH[5] = error code
+        mov $r14 0xc1c
+        shl b32 $r14 6
+        mov $r15 1
+        iowr I[$r14 + 0x000] $r15       // INTR_UP_SET
+        pop $r14
+        ret
+// HUB fuc initialisation, executed by triggering ucode start, will
+// fall through to main loop after completion.
+//
+// Input:
+//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
+//
+// Output:
+//   CC_SCRATCH[0]:
+//           31:31: set to signal completion
+//   CC_SCRATCH[1]:
+//            31:0: total PGRAPH context size
+//
+init:
+        clear b32 $r0
+        mov $sp $r0
+        mov $xdbase $r0
+        // enable fifo access
+        mov $r1 0x1200
+        mov $r2 2
+        iowr I[$r1 + 0x000] $r2 // FIFO_ENABLE
+        // setup i0 handler, and route all interrupts to it
+        mov $r1 ih
+        mov $iv0 $r1
+        mov $r1 0x400
+        iowr I[$r1 + 0x300] $r0 // INTR_DISPATCH
+        // route HUB_CHANNEL_SWITCH to fuc interrupt 8
+        mov $r3 0x404
+        shl b32 $r3 6
+        mov $r2 0x2003          // { HUB_CHANNEL_SWITCH, ZERO } -> intr 8
+        iowr I[$r3 + 0x000] $r2
+        // not sure what these are, route them because NVIDIA does, and
+        // the IRQ handler will signal the host if we ever get one.. we
+        // may find out if/why we need to handle these if so..
+        //
+        mov $r2 0x2004
+        iowr I[$r3 + 0x004] $r2 // { 0x04, ZERO } -> intr 9
+        mov $r2 0x200b
+        iowr I[$r3 + 0x008] $r2 // { 0x0b, ZERO } -> intr 10
+        mov $r2 0x200c
+        iowr I[$r3 + 0x01c] $r2 // { 0x0c, ZERO } -> intr 15
+        // enable all INTR_UP interrupts
+        mov $r2 0xc24
+        shl b32 $r2 6
+        not b32 $r3 $r0
+        iowr I[$r2] $r3
+        // enable fifo, ctxsw, 9, 10, 15 interrupts
+        mov $r2 -0x78fc         // 0x8704
+        sethi $r2 0
+        iowr I[$r1 + 0x000] $r2 // INTR_EN_SET
+        // fifo level triggered, rest edge
+        sub b32 $r1 0x100
+        mov $r2 4
+        iowr I[$r1] $r2
+        // enable interrupts
+        bset $flags ie0
+        // fetch enabled GPC/ROP counts
+        mov $r14 -0x69fc        // 0x409604
+        sethi $r14 0x400000
+        call nv_rd32
+        extr $r1 $r15 16:20
+        st b32 D[$r0 + rop_count] $r1
+        and $r15 0x1f
+        st b32 D[$r0 + gpc_count] $r15
+        // set BAR_REQMASK to GPC mask
+        mov $r1 1
+        shl b32 $r1 $r15
+        sub b32 $r1 1
+        mov $r2 0x40c
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r1
+        iowr I[$r2 + 0x100] $r1
+        // find context data for this chipset
+        mov $r2 0x800
+        shl b32 $r2 6
+        iord $r2 I[$r2 + 0x000]         // CC_SCRATCH[0]
+        mov $r15 chipsets - 8
+        init_find_chipset:
+                add b32 $r15 8
+                ld b32 $r3 D[$r15 + 0x00]
+                cmpu b32 $r3 $r2
+                bra e init_context
+                cmpu b32 $r3 0
+                bra ne init_find_chipset
+                // unknown chipset
+                ret
+        // context size calculation, reserve first 256 bytes for use by fuc
+        init_context:
+        mov $r1 256
+        // calculate size of mmio context data
+        ld b16 $r14 D[$r15 + 4]
+        ld b16 $r15 D[$r15 + 6]
+        sethi $r14 0
+        st b32 D[$r0 + hub_mmio_list_head] $r14
+        st b32 D[$r0 + hub_mmio_list_tail] $r15
+        call mmctx_size
+        // set mmctx base addresses now so we don't have to do it later,
+        // they don't (currently) ever change
+        mov $r3 0x700
+        shl b32 $r3 6
+        shr b32 $r4 $r1 8
+        iowr I[$r3 + 0x000] $r4         // MMCTX_SAVE_SWBASE
+        iowr I[$r3 + 0x100] $r4         // MMCTX_LOAD_SWBASE
+        add b32 $r3 0x1300
+        add b32 $r1 $r15
+        shr b32 $r15 2
+        iowr I[$r3 + 0x000] $r15        // MMCTX_LOAD_COUNT, wtf for?!?
+        // strands, base offset needs to be aligned to 256 bytes
+        shr b32 $r1 8
+        add b32 $r1 1
+        shl b32 $r1 8
+        mov b32 $r15 $r1
+        call strand_ctx_init
+        add b32 $r1 $r15
+        // initialise each GPC in sequence by passing in the offset of its
+        // context data in GPCn_CC_SCRATCH[1], and starting its FUC (which
+        // has previously been uploaded by the host) running.
+        //
+        // the GPC fuc init sequence will set GPCn_CC_SCRATCH[0] bit 31
+        // when it has completed, and return the size of its context data
+        // in GPCn_CC_SCRATCH[1]
+        //
+        ld b32 $r3 D[$r0 + gpc_count]
+        mov $r4 0x2000
+        sethi $r4 0x500000
+        init_gpc:
+                // setup, and start GPC ucode running
+                add b32 $r14 $r4 0x804
+                mov b32 $r15 $r1
+                call nv_wr32                    // CC_SCRATCH[1] = ctx offset
+                add b32 $r14 $r4 0x800
+                mov b32 $r15 $r2
+                call nv_wr32                    // CC_SCRATCH[0] = chipset
+                add b32 $r14 $r4 0x10c
+                clear b32 $r15
+                call nv_wr32
+                add b32 $r14 $r4 0x104
+                call nv_wr32                    // ENTRY
+                add b32 $r14 $r4 0x100
+                mov $r15 2                      // CTRL_START_TRIGGER
+                call nv_wr32                    // CTRL
+                // wait for it to complete, and adjust context size
+                add b32 $r14 $r4 0x800
+                init_gpc_wait:
+                        call nv_rd32
+                        xbit $r15 $r15 31
+                        bra e init_gpc_wait
+                add b32 $r14 $r4 0x804
+                call nv_rd32
+                add b32 $r1 $r15
+                // next!
+                add b32 $r4 0x8000
+                sub b32 $r3 1
+                bra ne init_gpc
+        // save context size, and tell host we're ready
+        mov $r2 0x800
+        shl b32 $r2 6
+        iowr I[$r2 + 0x100] $r1         // CC_SCRATCH[1]  = context size
+        add b32 $r2 0x800
+        clear b32 $r1
+        bset $r1 31
+        iowr I[$r2 + 0x000] $r1         // CC_SCRATCH[0] |= 0x80000000
+// Main program loop, very simple, sleeps until woken up by the interrupt
+// handler, pulls a command from the queue and executes its handler
+//
+main:
+        // sleep until we have something to do
+        bset $flags $p0
+        sleep $p0
+        mov $r13 cmd_queue
+        call queue_get
+        bra $p1 main
+        // context switch, requested by GPU?
+        cmpu b32 $r14 0x4001
+        bra ne main_not_ctx_switch
+                trace_set(T_AUTO)
+                mov $r1 0xb00
+                shl b32 $r1 6
+                iord $r2 I[$r1 + 0x100]         // CHAN_NEXT
+                iord $r1 I[$r1 + 0x000]         // CHAN_CUR
+                xbit $r3 $r1 31
+                bra e chsw_no_prev
+                        xbit $r3 $r2 31
+                        bra e chsw_prev_no_next
+                                push $r2
+                                mov b32 $r2 $r1
+                                trace_set(T_SAVE)
+                                bclr $flags $p1
+                                bset $flags $p2
+                                call ctx_xfer
+                                trace_clr(T_SAVE);
+                                pop $r2
+                                trace_set(T_LOAD);
+                                bset $flags $p1
+                                call ctx_xfer
+                                trace_clr(T_LOAD);
+                                bra chsw_done
+                        chsw_prev_no_next:
+                                push $r2
+                                mov b32 $r2 $r1
+                                bclr $flags $p1
+                                bclr $flags $p2
+                                call ctx_xfer
+                                pop $r2
+                                mov $r1 0xb00
+                                shl b32 $r1 6
+                                iowr I[$r1] $r2
+                                bra chsw_done
+                chsw_no_prev:
+                        xbit $r3 $r2 31
+                        bra e chsw_done
+                                bset $flags $p1
+                                bclr $flags $p2
+                                call ctx_xfer
+                // ack the context switch request
+                chsw_done:
+                mov $r1 0xb0c
+                shl b32 $r1 6
+                mov $r2 1
+                iowr I[$r1 + 0x000] $r2         // 0x409b0c
+                trace_clr(T_AUTO)
+                bra main
+        // request to set current channel? (*not* a context switch)
+        main_not_ctx_switch:
+        cmpu b32 $r14 0x0001
+        bra ne main_not_ctx_chan
+                mov b32 $r2 $r15
+                call ctx_chan
+                bra main_done
+        // request to store current channel context?
+        main_not_ctx_chan:
+        cmpu b32 $r14 0x0002
+        bra ne main_not_ctx_save
+                trace_set(T_SAVE)
+                bclr $flags $p1
+                bclr $flags $p2
+                call ctx_xfer
+                trace_clr(T_SAVE)
+                bra main_done
+        main_not_ctx_save:
+                shl b32 $r15 $r14 16
+                or $r15 E_BAD_COMMAND
+                call error
+                bra main
+        main_done:
+        mov $r1 0x820
+        shl b32 $r1 6
+        clear b32 $r2
+        bset $r2 31
+        iowr I[$r1 + 0x000] $r2         // CC_SCRATCH[0] |= 0x80000000
+        bra main
+// interrupt handler
+ih:
+        push $r8
+        mov $r8 $flags
+        push $r8
+        push $r9
+        push $r10
+        push $r11
+        push $r13
+        push $r14
+        push $r15
+        // incoming fifo command?
+        iord $r10 I[$r0 + 0x200]        // INTR
+        and $r11 $r10 0x00000004
+        bra e ih_no_fifo
+                // queue incoming fifo command for later processing
+                mov $r11 0x1900
+                mov $r13 cmd_queue
+                iord $r14 I[$r11 + 0x100]       // FIFO_CMD
+                iord $r15 I[$r11 + 0x000]       // FIFO_DATA
+                call queue_put
+                add b32 $r11 0x400
+                mov $r14 1
+                iowr I[$r11 + 0x000] $r14       // FIFO_ACK
+        // context switch request?
+        ih_no_fifo:
+        and $r11 $r10 0x00000100
+        bra e ih_no_ctxsw
+                // enqueue a context switch for later processing
+                mov $r13 cmd_queue
+                mov $r14 0x4001
+                call queue_put
+        // anything we didn't handle, bring it to the host's attention
+        ih_no_ctxsw:
+        mov $r11 0x104
+        not b32 $r11
+        and $r11 $r10 $r11
+        bra e ih_no_other
+                mov $r10 0xc1c
+                shl b32 $r10 6
+                iowr I[$r10] $r11       // INTR_UP_SET
+        // ack, and wake up main()
+        ih_no_other:
+        iowr I[$r0 + 0x100] $r10        // INTR_ACK
+        pop $r15
+        pop $r14
+        pop $r13
+        pop $r11
+        pop $r10
+        pop $r9
+        pop $r8
+        mov $flags $r8
+        pop $r8
+        bclr $flags $p0
+        iret
+// Not real sure, but, MEM_CMD 7 will hang forever if this isn't done
+ctx_4160s:
+        mov $r14 0x4160
+        sethi $r14 0x400000
+        mov $r15 1
+        call nv_wr32
+        ctx_4160s_wait:
+                call nv_rd32
+                xbit $r15 $r15 4
+                bra e ctx_4160s_wait
+        ret
+// Without clearing again at end of xfer, some things cause PGRAPH
+// to hang with STATUS=0x00000007 until it's cleared.. fbcon can
+// still function with it set however...
+ctx_4160c:
+        mov $r14 0x4160
+        sethi $r14 0x400000
+        clear b32 $r15
+        call nv_wr32
+        ret
+// Again, not real sure
+//
+// In: $r15 value to set 0x404170 to
+//
+ctx_4170s:
+        mov $r14 0x4170
+        sethi $r14 0x400000
+        or $r15 0x10
+        call nv_wr32
+        ret
+// Waits for a ctx_4170s() call to complete
+//
+ctx_4170w:
+        mov $r14 0x4170
+        sethi $r14 0x400000
+        call nv_rd32
+        and $r15 0x10
+        bra ne ctx_4170w
+        ret
+// Disables various things, waits a bit, and re-enables them..
+//
+// Not sure how exactly this helps, perhaps "ENABLE" is not such a
+// good description for the bits we turn off?  Anyways, without this,
+// funny things happen.
+//
+ctx_redswitch:
+        mov $r14 0x614
+        shl b32 $r14 6
+        mov $r15 0x270
+        iowr I[$r14] $r15       // HUB_RED_SWITCH = ENABLE_GPC, POWER_ALL
+        mov $r15 8
+        ctx_redswitch_delay:
+                sub b32 $r15 1
+                bra ne ctx_redswitch_delay
+        mov $r15 0x770
+        iowr I[$r14] $r15       // HUB_RED_SWITCH = ENABLE_ALL, POWER_ALL
+        ret
+// Not a clue what this is for, except that unless the value is 0x10, the
+// strand context is saved (and presumably restored) incorrectly..
+//
+// In: $r15 value to set to (0x00/0x10 are used)
+//
+ctx_86c:
+        mov $r14 0x86c
+        shl b32 $r14 6
+        iowr I[$r14] $r15       // HUB(0x86c) = val
+        mov $r14 -0x75ec
+        sethi $r14 0x400000
+        call nv_wr32            // ROP(0xa14) = val
+        mov $r14 -0x5794
+        sethi $r14 0x410000
+        call nv_wr32            // GPC(0x86c) = val
+        ret
+// ctx_load - load's a channel's ctxctl data, and selects its vm
+//
+// In: $r2 channel address
+//
+ctx_load:
+        trace_set(T_CHAN)
+        // switch to channel, somewhat magic in parts..
+        mov $r10 12             // DONE_UNK12
+        call wait_donez
+        mov $r1 0xa24
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r0 // 0x409a24
+        mov $r3 0xb00
+        shl b32 $r3 6
+        iowr I[$r3 + 0x100] $r2 // CHAN_NEXT
+        mov $r1 0xa0c
+        shl b32 $r1 6
+        mov $r4 7
+        iowr I[$r1 + 0x000] $r2 // MEM_CHAN
+        iowr I[$r1 + 0x100] $r4 // MEM_CMD
+        ctx_chan_wait_0:
+                iord $r4 I[$r1 + 0x100]
+                and $r4 0x1f
+                bra ne ctx_chan_wait_0
+        iowr I[$r3 + 0x000] $r2 // CHAN_CUR
+        // load channel header, fetch PGRAPH context pointer
+        mov $xtargets $r0
+        bclr $r2 31
+        shl b32 $r2 4
+        add b32 $r2 2
+        trace_set(T_LCHAN)
+        mov $r1 0xa04
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r2         // MEM_BASE
+        mov $r1 0xa20
+        shl b32 $r1 6
+        mov $r2 0x0002
+        sethi $r2 0x80000000
+        iowr I[$r1 + 0x000] $r2         // MEM_TARGET = vram
+        mov $r1 0x10                    // chan + 0x0210
+        mov $r2 xfer_data
+        sethi $r2 0x00020000            // 16 bytes
+        xdld $r1 $r2
+        xdwait
+        trace_clr(T_LCHAN)
+        // update current context
+        ld b32 $r1 D[$r0 + xfer_data + 4]
+        shl b32 $r1 24
+        ld b32 $r2 D[$r0 + xfer_data + 0]
+        shr b32 $r2 8
+        or $r1 $r2
+        st b32 D[$r0 + ctx_current] $r1
+        // set transfer base to start of context, and fetch context header
+        trace_set(T_LCTXH)
+        mov $r2 0xa04
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r1         // MEM_BASE
+        mov $r2 1
+        mov $r1 0xa20
+        shl b32 $r1 6
+        iowr I[$r1 + 0x000] $r2         // MEM_TARGET = vm
+        mov $r1 chan_data
+        sethi $r1 0x00060000            // 256 bytes
+        xdld $r0 $r1
+        xdwait
+        trace_clr(T_LCTXH)
+        trace_clr(T_CHAN)
+        ret
+// ctx_chan - handler for HUB_SET_CHAN command, will set a channel as
+//            the active channel for ctxctl, but not actually transfer
+//            any context data.  intended for use only during initial
+//            context construction.
+//
+// In: $r2 channel address
+//
+ctx_chan:
+        call ctx_4160s
+        call ctx_load
+        mov $r10 12                     // DONE_UNK12
+        call wait_donez
+        mov $r1 0xa10
+        shl b32 $r1 6
+        mov $r2 5
+        iowr I[$r1 + 0x000] $r2         // MEM_CMD = 5 (???)
+        ctx_chan_wait:
+                iord $r2 I[$r1 + 0x000]
+                or $r2 $r2
+                bra ne ctx_chan_wait
+        call ctx_4160c
+        ret
+// Execute per-context state overrides list
+//
+// Only executed on the first load of a channel.  Might want to look into
+// removing this and having the host directly modify the channel's context
+// to change this state...  The nouveau DRM already builds this list as
+// it's definitely needed for NVIDIA's, so we may as well use it for now
+//
+// Input: $r1 mmio list length
+//
+ctx_mmio_exec:
+        // set transfer base to be the mmio list
+        ld b32 $r3 D[$r0 + chan_mmio_address]
+        mov $r2 0xa04
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r3         // MEM_BASE
+        clear b32 $r3
+        ctx_mmio_loop:
+                // fetch next 256 bytes of mmio list if necessary
+                and $r4 $r3 0xff
+                bra ne ctx_mmio_pull
+                        mov $r5 xfer_data
+                        sethi $r5 0x00060000    // 256 bytes
+                        xdld $r3 $r5
+                        xdwait
+                // execute a single list entry
+                ctx_mmio_pull:
+                ld b32 $r14 D[$r4 + xfer_data + 0x00]
+                ld b32 $r15 D[$r4 + xfer_data + 0x04]
+                call nv_wr32
+                // next!
+                add b32 $r3 8
+                sub b32 $r1 1
+                bra ne ctx_mmio_loop
+        // set transfer base back to the current context
+        ctx_mmio_done:
+        ld b32 $r3 D[$r0 + ctx_current]
+        iowr I[$r2 + 0x000] $r3         // MEM_BASE
+        // disable the mmio list now, we don't need/want to execute it again
+        st b32 D[$r0 + chan_mmio_count] $r0
+        mov $r1 chan_data
+        sethi $r1 0x00060000            // 256 bytes
+        xdst $r0 $r1
+        xdwait
+        ret
+// Transfer HUB context data between GPU and storage area
+//
+// In: $r2 channel address
+//     $p1 clear on save, set on load
+//     $p2 set if opposite direction done/will be done, so:
+//              on save it means: "a load will follow this save"
+//              on load it means: "a save preceeded this load"
+//
+ctx_xfer:
+        bra not $p1 ctx_xfer_pre
+        bra $p2 ctx_xfer_pre_load
+        ctx_xfer_pre:
+                mov $r15 0x10
+                call ctx_86c
+                call ctx_4160s
+                bra not $p1 ctx_xfer_exec
+        ctx_xfer_pre_load:
+                mov $r15 2
+                call ctx_4170s
+                call ctx_4170w
+                call ctx_redswitch
+                clear b32 $r15
+                call ctx_4170s
+                call ctx_load
+        // fetch context pointer, and initiate xfer on all GPCs
+        ctx_xfer_exec:
+        ld b32 $r1 D[$r0 + ctx_current]
+        mov $r2 0x414
+        shl b32 $r2 6
+        iowr I[$r2 + 0x000] $r0 // BAR_STATUS = reset
+        mov $r14 -0x5b00
+        sethi $r14 0x410000
+        mov b32 $r15 $r1
+        call nv_wr32            // GPC_BCAST_WRCMD_DATA = ctx pointer
+        add b32 $r14 4
+        xbit $r15 $flags $p1
+        xbit $r2 $flags $p2
+        shl b32 $r2 1
+        or $r15 $r2
+        call nv_wr32            // GPC_BCAST_WRCMD_CMD = GPC_XFER(type)
+        // strands
+        mov $r1 0x4afc
+        sethi $r1 0x20000
+        mov $r2 0xc
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x0c
+        call strand_wait
+        mov $r2 0x47fc
+        sethi $r2 0x20000
+        iowr I[$r2] $r0         // STRAND_FIRST_GENE(0x3f) = 0x00
+        xbit $r2 $flags $p1
+        add b32 $r2 3
+        iowr I[$r1] $r2         // STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)
+        // mmio context
+        xbit $r10 $flags $p1    // direction
+        or $r10 6               // first, last
+        mov $r11 0              // base = 0
+        ld b32 $r12 D[$r0 + hub_mmio_list_head]
+        ld b32 $r13 D[$r0 + hub_mmio_list_tail]
+        mov $r14 0              // not multi
+        call mmctx_xfer
+        // wait for GPCs to all complete
+        mov $r10 8              // DONE_BAR
+        call wait_doneo
+        // wait for strand xfer to complete
+        call strand_wait
+        // post-op
+        bra $p1 ctx_xfer_post
+                mov $r10 12             // DONE_UNK12
+                call wait_donez
+                mov $r1 0xa10
+                shl b32 $r1 6
+                mov $r2 5
+                iowr I[$r1] $r2         // MEM_CMD
+                ctx_xfer_post_save_wait:
+                        iord $r2 I[$r1]
+                        or $r2 $r2
+                        bra ne ctx_xfer_post_save_wait
+        bra $p2 ctx_xfer_done
+        ctx_xfer_post:
+                mov $r15 2
+                call ctx_4170s
+                clear b32 $r15
+                call ctx_86c
+                call strand_post
+                call ctx_4170w
+                clear b32 $r15
+                call ctx_4170s
+                bra not $p1 ctx_xfer_no_post_mmio
+                ld b32 $r1 D[$r0 + chan_mmio_count]
+                or $r1 $r1
+                bra e ctx_xfer_no_post_mmio
+                        call ctx_mmio_exec
+                ctx_xfer_no_post_mmio:
+                call ctx_4160c
+        ctx_xfer_done:
+        ret
+.align 256
diff --git a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h
new file mode 100644
index 000000000000..b3b541b6d044
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h
@@ -0,0 +1,838 @@
+uint32_t nvc0_grhub_data[] = {
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x000000c0,
+        0x012c0090,
+        0x000000c1,
+        0x01300090,
+        0x000000c3,
+        0x012c0090,
+        0x000000c4,
+        0x012c0090,
+        0x000000c8,
+        0x012c0090,
+        0x000000ce,
+        0x012c0090,
+        0x00000000,
+        0x0417e91c,
+        0x04400204,
+        0x28404004,
+        0x00404044,
+        0x34404094,
+        0x184040d0,
+        0x004040f8,
+        0x08404130,
+        0x08404150,
+        0x04404164,
+        0x08404174,
+        0x1c404200,
+        0x34404404,
+        0x0c404460,
+        0x00404480,
+        0x00404498,
+        0x0c404604,
+        0x7c404618,
+        0x50404698,
+        0x044046f0,
+        0x54404700,
+        0x00405800,
+        0x08405830,
+        0x00405854,
+        0x0c405870,
+        0x04405a00,
+        0x00405a18,
+        0x00406020,
+        0x0c406028,
+        0x044064a8,
+        0x044064b4,
+        0x00407804,
+        0x1440780c,
+        0x004078bc,
+        0x18408000,
+        0x00408064,
+        0x08408800,
+        0x0c408900,
+        0x00408980,
+        0x044064c0,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+uint32_t nvc0_grhub_code[] = {
+        0x03090ef5,
+        0x9800d898,
+        0x86f001d9,
+        0x0489b808,
+        0xf00c1bf4,
+        0x21f502f7,
+        0x00f802ec,
+        0xb60798c4,
+        0x8dbb0384,
+        0x0880b600,
+        0x80008e80,
+        0x90b6018f,
+        0x0f94f001,
+        0xf801d980,
+        0x0131f400,
+        0x9800d898,
+        0x89b801d9,
+        0x210bf404,
+        0xb60789c4,
+        0x9dbb0394,
+        0x0890b600,
+        0x98009e98,
+        0x80b6019f,
+        0x0f84f001,
+        0xf400d880,
+        0x00f80132,
+        0x0728b7f1,
+        0xb906b4b6,
+        0xc9f002ec,
+        0x00bcd01f,
+        0xc800bccf,
+        0x1bf41fcc,
+        0x06a7f0fa,
+        0x010321f5,
+        0xf840bfcf,
+        0x28b7f100,
+        0x06b4b607,
+        0xb980bfd0,
+        0xc9f002ec,
+        0x1ec9f01f,
+        0xcf00bcd0,
+        0xccc800bc,
+        0xfa1bf41f,
+        0x87f100f8,
+        0x84b60430,
+        0x1ff9f006,
+        0xf8008fd0,
+        0x3087f100,
+        0x0684b604,
+        0xf80080d0,
+        0x3c87f100,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d000,
+        0x081887f1,
+        0xd00684b6,
+        0x87f1008a,
+        0x84b60400,
+        0x0088cf06,
+        0xf4888aff,
+        0x87f1f31b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00099,
+        0xf100f800,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00099f0,
+        0x87f10089,
+        0x84b60818,
+        0x008ad006,
+        0x040087f1,
+        0xcf0684b6,
+        0x8aff0088,
+        0xf30bf488,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0099f094,
+        0xf80089d0,
+        0x9894bd00,
+        0x85b600e8,
+        0x0180b61a,
+        0xbb0284b6,
+        0xe0b60098,
+        0x04efb804,
+        0xb9eb1bf4,
+        0x00f8029f,
+        0x083c87f1,
+        0xbd0684b6,
+        0x0199f094,
+        0xf10089d0,
+        0xb6071087,
+        0x94bd0684,
+        0xf405bbfd,
+        0x8bd0090b,
+        0x0099f000,
+        0xf405eefd,
+        0x8ed00c0b,
+        0xc08fd080,
+        0xb70199f0,
+        0xc8010080,
+        0xb4b600ab,
+        0x0cb9f010,
+        0xb601aec8,
+        0xbefd11e4,
+        0x008bd005,
+        0xf0008ecf,
+        0x0bf41fe4,
+        0x00ce98fa,
+        0xd005e9fd,
+        0xc0b6c08e,
+        0x04cdb804,
+        0xc8e81bf4,
+        0x1bf402ab,
+        0x008bcf18,
+        0xb01fb4f0,
+        0x1bf410b4,
+        0x02a7f0f7,
+        0xf4c921f4,
+        0xabc81b0e,
+        0x10b4b600,
+        0xf00cb9f0,
+        0x8bd012b9,
+        0x008bcf00,
+        0xf412bbc8,
+        0x87f1fa1b,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00199,
+        0xf900f800,
+        0x02a7f0a0,
+        0xfcc921f4,
+        0xf100f8a0,
+        0xf04afc87,
+        0x97f00283,
+        0x0089d00c,
+        0x020721f5,
+        0x87f100f8,
+        0x83f04afc,
+        0x0d97f002,
+        0xf50089d0,
+        0xf8020721,
+        0xfca7f100,
+        0x02a3f04f,
+        0x0500aba2,
+        0xd00fc7f0,
+        0xc7f000ac,
+        0x00bcd00b,
+        0x020721f5,
+        0xf000aed0,
+        0xbcd00ac7,
+        0x0721f500,
+        0xf100f802,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x21f50089,
+        0xe7f00213,
+        0x3921f503,
+        0xfca7f102,
+        0x02a3f046,
+        0x0400aba0,
+        0xf040a0d0,
+        0xbcd001c7,
+        0x0721f500,
+        0x010c9202,
+        0xf000acd0,
+        0xbcd002c7,
+        0x0721f500,
+        0x2621f502,
+        0x8087f102,
+        0x0684b608,
+        0xb70089cf,
+        0x95220080,
+        0x8ed008fe,
+        0x408ed000,
+        0xb6808acf,
+        0xa0b606a5,
+        0x00eabb01,
+        0xb60480b6,
+        0x1bf40192,
+        0x08e4b6e8,
+        0xf1f2efbc,
+        0xb6085c87,
+        0x94bd0684,
+        0xd00399f0,
+        0x00f80089,
+        0xe7f1e0f9,
+        0xe4b60814,
+        0x00efd006,
+        0x0c1ce7f1,
+        0xf006e4b6,
+        0xefd001f7,
+        0xf8e0fc00,
+        0xfe04bd00,
+        0x07fe0004,
+        0x0017f100,
+        0x0227f012,
+        0xf10012d0,
+        0xfe05b917,
+        0x17f10010,
+        0x10d00400,
+        0x0437f1c0,
+        0x0634b604,
+        0x200327f1,
+        0xf10032d0,
+        0xd0200427,
+        0x27f10132,
+        0x32d0200b,
+        0x0c27f102,
+        0x0732d020,
+        0x0c2427f1,
+        0xb90624b6,
+        0x23d00003,
+        0x0427f100,
+        0x0023f087,
+        0xb70012d0,
+        0xf0010012,
+        0x12d00427,
+        0x1031f400,
+        0x9604e7f1,
+        0xf440e3f0,
+        0xf1c76821,
+        0x01018090,
+        0x801ff4f0,
+        0x17f0000f,
+        0x041fbb01,
+        0xf10112b6,
+        0xb6040c27,
+        0x21d00624,
+        0x4021d000,
+        0x080027f1,
+        0xcf0624b6,
+        0xf7f00022,
+        0x08f0b654,
+        0xb800f398,
+        0x0bf40432,
+        0x0034b00b,
+        0xf8f11bf4,
+        0x0017f100,
+        0x02fe5801,
+        0xf003ff58,
+        0x0e8000e3,
+        0x150f8014,
+        0x013d21f5,
+        0x070037f1,
+        0x950634b6,
+        0x34d00814,
+        0x4034d000,
+        0x130030b7,
+        0xb6001fbb,
+        0x3fd002f5,
+        0x0815b600,
+        0xb60110b6,
+        0x1fb90814,
+        0x6321f502,
+        0x001fbb02,
+        0xf1000398,
+        0xf0200047,
+        0x4ea05043,
+        0x1fb90804,
+        0x8d21f402,
+        0x08004ea0,
+        0xf4022fb9,
+        0x4ea08d21,
+        0xf4bd010c,
+        0xa08d21f4,
+        0xf401044e,
+        0x4ea08d21,
+        0xf7f00100,
+        0x8d21f402,
+        0x08004ea0,
+        0xc86821f4,
+        0x0bf41fff,
+        0x044ea0fa,
+        0x6821f408,
+        0xb7001fbb,
+        0xb6800040,
+        0x1bf40132,
+        0x0027f1b4,
+        0x0624b608,
+        0xb74021d0,
+        0xbd080020,
+        0x1f19f014,
+        0xf40021d0,
+        0x28f40031,
+        0x08d7f000,
+        0xf43921f4,
+        0xe4b1f401,
+        0x1bf54001,
+        0x87f100d1,
+        0x84b6083c,
+        0xf094bd06,
+        0x89d00499,
+        0x0017f100,
+        0x0614b60b,
+        0xcf4012cf,
+        0x13c80011,
+        0x7e0bf41f,
+        0xf41f23c8,
+        0x20f95a0b,
+        0xf10212b9,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00799f0,
+        0x32f40089,
+        0x0231f401,
+        0x082921f5,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0799f094,
+        0xfc0089d0,
+        0x3c87f120,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d006,
+        0xf50131f4,
+        0xf1082921,
+        0xb6085c87,
+        0x94bd0684,
+        0xd00699f0,
+        0x0ef40089,
+        0xb920f931,
+        0x32f40212,
+        0x0232f401,
+        0x082921f5,
+        0x17f120fc,
+        0x14b60b00,
+        0x0012d006,
+        0xc8130ef4,
+        0x0bf41f23,
+        0x0131f40d,
+        0xf50232f4,
+        0xf1082921,
+        0xb60b0c17,
+        0x27f00614,
+        0x0012d001,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0499f094,
+        0xf50089d0,
+        0xb0ff200e,
+        0x1bf401e4,
+        0x02f2b90d,
+        0x07b521f5,
+        0xb0420ef4,
+        0x1bf402e4,
+        0x3c87f12e,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d007,
+        0xf40132f4,
+        0x21f50232,
+        0x87f10829,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00799,
+        0x110ef400,
+        0xf010ef94,
+        0x21f501f5,
+        0x0ef502ec,
+        0x17f1fed1,
+        0x14b60820,
+        0xf024bd06,
+        0x12d01f29,
+        0xbe0ef500,
+        0xfe80f9fe,
+        0x80f90188,
+        0xa0f990f9,
+        0xd0f9b0f9,
+        0xf0f9e0f9,
+        0xc4800acf,
+        0x0bf404ab,
+        0x00b7f11d,
+        0x08d7f019,
+        0xcf40becf,
+        0x21f400bf,
+        0x00b0b704,
+        0x01e7f004,
+        0xe400bed0,
+        0xf40100ab,
+        0xd7f00d0b,
+        0x01e7f108,
+        0x0421f440,
+        0x0104b7f1,
+        0xabffb0bd,
+        0x0d0bf4b4,
+        0x0c1ca7f1,
+        0xd006a4b6,
+        0x0ad000ab,
+        0xfcf0fc40,
+        0xfcd0fce0,
+        0xfca0fcb0,
+        0xfe80fc90,
+        0x80fc0088,
+        0xf80032f4,
+        0x60e7f101,
+        0x40e3f041,
+        0xf401f7f0,
+        0x21f48d21,
+        0x04ffc868,
+        0xf8fa0bf4,
+        0x60e7f100,
+        0x40e3f041,
+        0x21f4f4bd,
+        0xf100f88d,
+        0xf04170e7,
+        0xf5f040e3,
+        0x8d21f410,
+        0xe7f100f8,
+        0xe3f04170,
+        0x6821f440,
+        0xf410f4f0,
+        0x00f8f31b,
+        0x0614e7f1,
+        0xf106e4b6,
+        0xd00270f7,
+        0xf7f000ef,
+        0x01f2b608,
+        0xf1fd1bf4,
+        0xd00770f7,
+        0x00f800ef,
+        0x086ce7f1,
+        0xd006e4b6,
+        0xe7f100ef,
+        0xe3f08a14,
+        0x8d21f440,
+        0xa86ce7f1,
+        0xf441e3f0,
+        0x00f88d21,
+        0x083c87f1,
+        0xbd0684b6,
+        0x0599f094,
+        0xf00089d0,
+        0x21f40ca7,
+        0x2417f1c9,
+        0x0614b60a,
+        0xf10010d0,
+        0xb60b0037,
+        0x32d00634,
+        0x0c17f140,
+        0x0614b60a,
+        0xd00747f0,
+        0x14d00012,
+        0x4014cf40,
+        0xf41f44f0,
+        0x32d0fa1b,
+        0x000bfe00,
+        0xb61f2af0,
+        0x20b60424,
+        0x3c87f102,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d008,
+        0x0a0417f1,
+        0xd00614b6,
+        0x17f10012,
+        0x14b60a20,
+        0x0227f006,
+        0x800023f1,
+        0xf00012d0,
+        0x27f11017,
+        0x23f00300,
+        0x0512fa02,
+        0x87f103f8,
+        0x84b6085c,
+        0xf094bd06,
+        0x89d00899,
+        0xc1019800,
+        0x981814b6,
+        0x25b6c002,
+        0x0512fd08,
+        0xf1160180,
+        0xb6083c87,
+        0x94bd0684,
+        0xd00999f0,
+        0x27f10089,
+        0x24b60a04,
+        0x0021d006,
+        0xf10127f0,
+        0xb60a2017,
+        0x12d00614,
+        0x0017f100,
+        0x0613f002,
+        0xf80501fa,
+        0x5c87f103,
+        0x0684b608,
+        0x99f094bd,
+        0x0089d009,
+        0x085c87f1,
+        0xbd0684b6,
+        0x0599f094,
+        0xf80089d0,
+        0x3121f500,
+        0xb821f506,
+        0x0ca7f006,
+        0xf1c921f4,
+        0xb60a1017,
+        0x27f00614,
+        0x0012d005,
+        0xfd0012cf,
+        0x1bf40522,
+        0x4921f5fa,
+        0x9800f806,
+        0x27f18103,
+        0x24b60a04,
+        0x0023d006,
+        0x34c434bd,
+        0x0f1bf4ff,
+        0x030057f1,
+        0xfa0653f0,
+        0x03f80535,
+        0x98c04e98,
+        0x21f4c14f,
+        0x0830b68d,
+        0xf40112b6,
+        0x0398df1b,
+        0x0023d016,
+        0xf1800080,
+        0xf0020017,
+        0x01fa0613,
+        0xf803f806,
+        0x0611f400,
+        0xf01102f4,
+        0x21f510f7,
+        0x21f50698,
+        0x11f40631,
+        0x02f7f01c,
+        0x065721f5,
+        0x066621f5,
+        0x067821f5,
+        0x21f5f4bd,
+        0x21f50657,
+        0x019806b8,
+        0x1427f116,
+        0x0624b604,
+        0xf10020d0,
+        0xf0a500e7,
+        0x1fb941e3,
+        0x8d21f402,
+        0xf004e0b6,
+        0x2cf001fc,
+        0x0124b602,
+        0xf405f2fd,
+        0x17f18d21,
+        0x13f04afc,
+        0x0c27f002,
+        0xf50012d0,
+        0xf1020721,
+        0xf047fc27,
+        0x20d00223,
+        0x012cf000,
+        0xd00320b6,
+        0xacf00012,
+        0x06a5f001,
+        0x9800b7f0,
+        0x0d98140c,
+        0x00e7f015,
+        0x015c21f5,
+        0xf508a7f0,
+        0xf5010321,
+        0xf4020721,
+        0xa7f02201,
+        0xc921f40c,
+        0x0a1017f1,
+        0xf00614b6,
+        0x12d00527,
+        0x0012cf00,
+        0xf40522fd,
+        0x02f4fa1b,
+        0x02f7f032,
+        0x065721f5,
+        0x21f5f4bd,
+        0x21f50698,
+        0x21f50226,
+        0xf4bd0666,
+        0x065721f5,
+        0x981011f4,
+        0x11fd8001,
+        0x070bf405,
+        0x07df21f5,
+        0x064921f5,
+        0x000000f8,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c
index 82357d2df1f4..b701c439c92e 100644
--- a/drivers/gpu/drm/nouveau/nvc0_instmem.c
+++ b/drivers/gpu/drm/nouveau/nvc0_instmem.c
@@ -32,7 +32,6 @@ struct nvc0_instmem_priv {
        struct nouveau_channel *bar1;
        struct nouveau_gpuobj  *bar3_pgd;
        struct nouveau_channel *bar3;
-        struct nouveau_gpuobj  *chan_pgd;
 };
 int
@@ -181,17 +180,11 @@ nvc0_instmem_init(struct drm_device *dev)
                goto error;
        /* channel vm */
-        ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL, &vm);
+        ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL,
+                             &dev_priv->chan_vm);
        if (ret)
                goto error;
-        ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096, 0, &priv->chan_pgd);
-        if (ret)
-                goto error;
-        nouveau_vm_ref(vm, &dev_priv->chan_vm, priv->chan_pgd);
-        nouveau_vm_ref(NULL, &vm, NULL);
        nvc0_instmem_resume(dev);
        return 0;
 error:
@@ -211,8 +204,7 @@ nvc0_instmem_takedown(struct drm_device *dev)
        nv_wr32(dev, 0x1704, 0x00000000);
        nv_wr32(dev, 0x1714, 0x00000000);
-        nouveau_vm_ref(NULL, &dev_priv->chan_vm, priv->chan_pgd);
+        nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
-        nouveau_gpuobj_ref(NULL, &priv->chan_pgd);
        nvc0_channel_del(&priv->bar1);
        nouveau_vm_ref(NULL, &dev_priv->bar1_vm, priv->bar1_pgd);
diff --git a/drivers/gpu/drm/nouveau/nvc0_vm.c b/drivers/gpu/drm/nouveau/nvc0_vm.c
index a179e6c55afb..9e352944a35a 100644
--- a/drivers/gpu/drm/nouveau/nvc0_vm.c
+++ b/drivers/gpu/drm/nouveau/nvc0_vm.c
@@ -105,7 +105,11 @@ nvc0_vm_flush(struct nouveau_vm *vm)
        struct drm_device *dev = vm->dev;
        struct nouveau_vm_pgd *vpgd;
        unsigned long flags;
-        u32 engine = (dev_priv->chan_vm == vm) ? 1 : 5;
+        u32 engine;
+        engine = 1;
+        if (vm == dev_priv->bar1_vm || vm == dev_priv->bar3_vm)
+                engine |= 4;
        pinstmem->flush(vm->dev);
diff --git a/drivers/gpu/drm/nouveau/nvc0_vram.c b/drivers/gpu/drm/nouveau/nvc0_vram.c
index 67c6ec6f34ea..e45a24d84e98 100644
--- a/drivers/gpu/drm/nouveau/nvc0_vram.c
+++ b/drivers/gpu/drm/nouveau/nvc0_vram.c
@@ -61,9 +61,7 @@ nvc0_vram_new(struct drm_device *dev, u64 size, u32 align, u32 ncmin,
              u32 type, struct nouveau_mem **pmem)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
-        struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
+        struct nouveau_mm *mm = dev_priv->engine.vram.mm;
-        struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
-        struct nouveau_mm *mm = man->priv;
        struct nouveau_mm_node *r;
        struct nouveau_mem *mem;
        int ret;
@@ -105,9 +103,15 @@ int
 nvc0_vram_init(struct drm_device *dev)
 {
        struct drm_nouveau_private *dev_priv = dev->dev_private;
+        struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
+        const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+        const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+        u32 length;
        dev_priv->vram_size  = nv_rd32(dev, 0x10f20c) << 20;
        dev_priv->vram_size *= nv_rd32(dev, 0x121c74);
-        dev_priv->vram_rblock_size = 4096;
-        return 0;
+        length = (dev_priv->vram_size >> 12) - rsvd_head - rsvd_tail;
+        return nouveau_mm_init(&vram->mm, rsvd_head, length, 1);
 }
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index 23d36417158d..189e86522b5b 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -856,7 +856,6 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
        case SQ_PGM_START_PS:
        case SQ_PGM_START_HS:
        case SQ_PGM_START_LS:
-        case GDS_ADDR_BASE:
        case SQ_CONST_MEM_BASE:
        case SQ_ALU_CONST_CACHE_GS_0:
        case SQ_ALU_CONST_CACHE_GS_1:
@@ -946,6 +945,34 @@ static inline int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u3
                }
                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                break;
+        case SX_MEMORY_EXPORT_BASE:
+                if (p->rdev->family >= CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONFIG_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONFIG_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                break;
+        case CAYMAN_SX_SCATTER_EXPORT_BASE:
+                if (p->rdev->family < CHIP_CAYMAN) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                 "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONTEXT_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                break;
        default:
                dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                return -EINVAL;
@@ -1153,6 +1180,34 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                        return r;
                }
                break;
+        case PACKET3_DISPATCH_DIRECT:
+                if (pkt->count != 3) {
+                        DRM_ERROR("bad DISPATCH_DIRECT\n");
+                        return -EINVAL;
+                }
+                r = evergreen_cs_track_check(p);
+                if (r) {
+                        dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx);
+                        return r;
+                }
+                break;
+        case PACKET3_DISPATCH_INDIRECT:
+                if (pkt->count != 1) {
+                        DRM_ERROR("bad DISPATCH_INDIRECT\n");
+                        return -EINVAL;
+                }
+                r = evergreen_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        DRM_ERROR("bad DISPATCH_INDIRECT\n");
+                        return -EINVAL;
+                }
+                ib[idx+0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
+                r = evergreen_cs_track_check(p);
+                if (r) {
+                        dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
+                        return r;
+                }
+                break;
        case PACKET3_WAIT_REG_MEM:
                if (pkt->count != 5) {
                        DRM_ERROR("bad WAIT_REG_MEM\n");
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index b7b2714f0b32..7363d9dec909 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -351,6 +351,7 @@
 #define         COLOR_BUFFER_SIZE(x)                            ((x) << 0)
 #define         POSITION_BUFFER_SIZE(x)                         ((x) << 8)
 #define         SMX_BUFFER_SIZE(x)                              ((x) << 16)
+#define SX_MEMORY_EXPORT_BASE                           0x9010
 #define SX_MISC                                         0x28350
 #define CB_PERF_CTR0_SEL_0                              0x9A20
@@ -1122,6 +1123,7 @@
 #define CAYMAN_PA_SC_AA_CONFIG                          0x28BE0
 #define         CAYMAN_MSAA_NUM_SAMPLES_SHIFT           0
 #define         CAYMAN_MSAA_NUM_SAMPLES_MASK            0x7
+#define CAYMAN_SX_SCATTER_EXPORT_BASE                   0x28358
 /* cayman packet3 addition */
 #define CAYMAN_PACKET3_DEALLOC_STATE                    0x14
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 909bda8dd550..db8ef1905d5f 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -1200,6 +1200,15 @@ static inline int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx
                }
                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
                break;
+        case SX_MEMORY_EXPORT_BASE:
+                r = r600_cs_packet_next_reloc(p, &reloc);
+                if (r) {
+                        dev_warn(p->dev, "bad SET_CONFIG_REG "
+                                        "0x%04X\n", reg);
+                        return -EINVAL;
+                }
+                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                break;
        default:
                dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx);
                return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index b2449629537d..df8218bb83a6 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -625,7 +625,7 @@ static struct radeon_asic r600_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &r600_cs_parse,
        .copy_blit = &r600_copy_blit,
-        .copy_dma = &r600_copy_blit,
+        .copy_dma = NULL,
        .copy = &r600_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -672,7 +672,7 @@ static struct radeon_asic rs780_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &r600_cs_parse,
        .copy_blit = &r600_copy_blit,
-        .copy_dma = &r600_copy_blit,
+        .copy_dma = NULL,
        .copy = &r600_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -719,7 +719,7 @@ static struct radeon_asic rv770_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &r600_cs_parse,
        .copy_blit = &r600_copy_blit,
-        .copy_dma = &r600_copy_blit,
+        .copy_dma = NULL,
        .copy = &r600_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -766,7 +766,7 @@ static struct radeon_asic evergreen_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &evergreen_cs_parse,
        .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
        .copy = &evergreen_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -813,7 +813,7 @@ static struct radeon_asic sumo_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &evergreen_cs_parse,
        .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
        .copy = &evergreen_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -860,7 +860,7 @@ static struct radeon_asic btc_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &evergreen_cs_parse,
        .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
        .copy = &evergreen_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
@@ -907,7 +907,7 @@ static struct radeon_asic cayman_asic = {
        .fence_ring_emit = &r600_fence_ring_emit,
        .cs_parse = &evergreen_cs_parse,
        .copy_blit = &evergreen_copy_blit,
-        .copy_dma = &evergreen_copy_blit,
+        .copy_dma = NULL,
        .copy = &evergreen_copy_blit,
        .get_engine_clock = &radeon_atom_get_engine_clock,
        .set_engine_clock = &radeon_atom_set_engine_clock,
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index 75867792a4e2..045ec59478f9 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -2115,7 +2115,7 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
        if (drm_pci_device_is_agp(dev))
                dev_priv->flags |= RADEON_IS_AGP;
-        else if (drm_pci_device_is_pcie(dev))
+        else if (pci_is_pcie(dev->pdev))
                dev_priv->flags |= RADEON_IS_PCIE;
        else
                dev_priv->flags |= RADEON_IS_PCI;
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 292f73f0ddbd..28f4655905bc 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -282,7 +282,7 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
        spin_lock_irqsave(&rdev->ddev->event_lock, flags);
        work = radeon_crtc->unpin_work;
        if (work == NULL ||
-            !radeon_fence_signaled(work->fence)) {
+            (work->fence && !radeon_fence_signaled(work->fence))) {
                spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
                return;
        }
@@ -348,7 +348,6 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
        struct radeon_framebuffer *new_radeon_fb;
        struct drm_gem_object *obj;
        struct radeon_bo *rbo;
-        struct radeon_fence *fence;
        struct radeon_unpin_work *work;
        unsigned long flags;
        u32 tiling_flags, pitch_pixels;
@@ -359,16 +358,9 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
        if (work == NULL)
                return -ENOMEM;
-        r = radeon_fence_create(rdev, &fence);
-        if (unlikely(r != 0)) {
-                kfree(work);
-                DRM_ERROR("flip queue: failed to create fence.\n");
-                return -ENOMEM;
-        }
        work->event = event;
        work->rdev = rdev;
        work->crtc_id = radeon_crtc->crtc_id;
-        work->fence = radeon_fence_ref(fence);
        old_radeon_fb = to_radeon_framebuffer(crtc->fb);
        new_radeon_fb = to_radeon_framebuffer(fb);
        /* schedule unpin of the old buffer */
@@ -377,6 +369,10 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
        drm_gem_object_reference(obj);
        rbo = gem_to_radeon_bo(obj);
        work->old_rbo = rbo;
+        obj = new_radeon_fb->obj;
+        rbo = gem_to_radeon_bo(obj);
+        if (rbo->tbo.sync_obj)
+                work->fence = radeon_fence_ref(rbo->tbo.sync_obj);
        INIT_WORK(&work->work, radeon_unpin_work_func);
        /* We borrow the event spin lock for protecting unpin_work */
@@ -391,9 +387,6 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
        spin_unlock_irqrestore(&dev->event_lock, flags);
        /* pin the new buffer */
-        obj = new_radeon_fb->obj;
-        rbo = gem_to_radeon_bo(obj);
        DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",
                         work->old_rbo, rbo);
@@ -461,37 +454,18 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
                goto pflip_cleanup1;
        }
-        /* 32 ought to cover us */
-        r = radeon_ring_lock(rdev, 32);
-        if (r) {
-                DRM_ERROR("failed to lock the ring before flip\n");
-                goto pflip_cleanup2;
-        }
-        /* emit the fence */
-        radeon_fence_emit(rdev, fence);
        /* set the proper interrupt */
        radeon_pre_page_flip(rdev, radeon_crtc->crtc_id);
-        /* fire the ring */
-        radeon_ring_unlock_commit(rdev);
        return 0;
-pflip_cleanup2:
-        drm_vblank_put(dev, radeon_crtc->crtc_id);
 pflip_cleanup1:
-        r = radeon_bo_reserve(rbo, false);
+        if (unlikely(radeon_bo_reserve(rbo, false) != 0)) {
-        if (unlikely(r != 0)) {
                DRM_ERROR("failed to reserve new rbo in error path\n");
                goto pflip_cleanup;
        }
-        r = radeon_bo_unpin(rbo);
+        if (unlikely(radeon_bo_unpin(rbo) != 0)) {
-        if (unlikely(r != 0)) {
-                radeon_bo_unreserve(rbo);
-                r = -EINVAL;
                DRM_ERROR("failed to unpin new rbo in error path\n");
-                goto pflip_cleanup;
        }
        radeon_bo_unreserve(rbo);
@@ -501,7 +475,7 @@ pflip_cleanup:
 unlock_free:
        drm_gem_object_unreference_unlocked(old_radeon_fb->obj);
        spin_unlock_irqrestore(&dev->event_lock, flags);
-        radeon_fence_unref(&fence);
+        radeon_fence_unref(&work->fence);
        kfree(work);
        return r;
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 73dfbe8e5f9e..cbb4584a4a23 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -50,7 +50,7 @@
 *   2.7.0 - fixups for r600 2D tiling support. (no external ABI change), add eg dyn gpr regs
 *   2.8.0 - pageflip support, r500 US_FORMAT regs. r500 ARGB2101010 colorbuf, r300->r500 CMASK, clock crystal query
 *   2.9.0 - r600 tiling (s3tc,rgtc) working, SET_PREDICATION packet 3 on r600 + eg, backend query
- *   2.10.0 - fusion 2D tiling
+ *   2.10.0 - fusion 2D tiling, initial compute support for the CS checker
 */
 #define KMS_DRIVER_MAJOR        2
 #define KMS_DRIVER_MINOR        10
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index bd58af658581..6f80a216bb39 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -60,7 +60,7 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
        /* update BUS flag */
        if (drm_pci_device_is_agp(dev)) {
                flags |= RADEON_IS_AGP;
-        } else if (drm_pci_device_is_pcie(dev)) {
+        } else if (pci_is_pcie(dev->pdev)) {
                flags |= RADEON_IS_PCIE;
        } else {
                flags |= RADEON_IS_PCI;
diff --git a/drivers/gpu/drm/radeon/reg_srcs/cayman b/drivers/gpu/drm/radeon/reg_srcs/cayman
index 0aa8e85a9457..2316977eb924 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/cayman
+++ b/drivers/gpu/drm/radeon/reg_srcs/cayman
@@ -208,6 +208,7 @@ cayman 0x9400
 0x0002834C PA_SC_VPORT_ZMAX_15
 0x00028350 SX_MISC
 0x00028354 SX_SURFACE_SYNC
+0x0002835C SX_SCATTER_EXPORT_SIZE
 0x00028380 SQ_VTX_SEMANTIC_0
 0x00028384 SQ_VTX_SEMANTIC_1
 0x00028388 SQ_VTX_SEMANTIC_2
@@ -432,6 +433,7 @@ cayman 0x9400
 0x00028700 SPI_STACK_MGMT
 0x00028704 SPI_WAVE_MGMT_1
 0x00028708 SPI_WAVE_MGMT_2
+0x00028720 GDS_ADDR_BASE
 0x00028724 GDS_ADDR_SIZE
 0x00028780 CB_BLEND0_CONTROL
 0x00028784 CB_BLEND1_CONTROL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/evergreen b/drivers/gpu/drm/radeon/reg_srcs/evergreen
index 0e28cae7ea43..161737a28c23 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/evergreen
+++ b/drivers/gpu/drm/radeon/reg_srcs/evergreen
@@ -44,6 +44,7 @@ evergreen 0x9400
 0x00008E28 SQ_STATIC_THREAD_MGMT_3
 0x00008E2C SQ_LDS_RESOURCE_MGMT
 0x00008E48 SQ_EX_ALLOC_TABLE_SLOTS
+0x00009014 SX_MEMORY_EXPORT_SIZE
 0x00009100 SPI_CONFIG_CNTL
 0x0000913C SPI_CONFIG_CNTL_1
 0x00009508 TA_CNTL_AUX
@@ -442,7 +443,9 @@ evergreen 0x9400
 0x000286EC SPI_COMPUTE_NUM_THREAD_X
 0x000286F0 SPI_COMPUTE_NUM_THREAD_Y
 0x000286F4 SPI_COMPUTE_NUM_THREAD_Z
+0x00028720 GDS_ADDR_BASE
 0x00028724 GDS_ADDR_SIZE
+0x00028728 GDS_ORDERED_WAVE_PER_SE
 0x00028780 CB_BLEND0_CONTROL
 0x00028784 CB_BLEND1_CONTROL
 0x00028788 CB_BLEND2_CONTROL
diff --git a/drivers/gpu/drm/radeon/reg_srcs/r600 b/drivers/gpu/drm/radeon/reg_srcs/r600
index ea49752ee99c..0380c5c15f80 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/r600
+++ b/drivers/gpu/drm/radeon/reg_srcs/r600
@@ -429,6 +429,7 @@ r600 0x9400
 0x00028438 SX_ALPHA_REF
 0x00028410 SX_ALPHA_TEST_CONTROL
 0x00028350 SX_MISC
+0x00009014 SX_MEMORY_EXPORT_SIZE
 0x00009604 TC_INVALIDATE
 0x00009400 TD_FILTER4
 0x00009404 TD_FILTER4_1
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index d948575717bf..170e751c283e 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -355,7 +355,7 @@ restart:
                        if (nr_free)
                                goto restart;
-                        /* Not allowed to fall tough or break because
+                        /* Not allowed to fall through or break because
                         * following context is inside spinlock while we are
                         * outside here.
                         */
@@ -556,7 +556,7 @@ out:
 }
 /**
- * Fill the given pool if there isn't enough pages and requested number of
+ * Fill the given pool if there aren't enough pages and the requested number of
 * pages is small.
 */
 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
@@ -576,8 +576,8 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
        pool->fill_lock = true;
-        /* If allocation request is small and there is not enough
+        /* If allocation request is small and there are not enough
-         * pages in pool we fill the pool first */
+         * pages in a pool we fill the pool up first. */
        if (count < _manager->options.small
                && count > pool->npages) {
                struct list_head new_pages;
@@ -614,9 +614,9 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 }
 /**
- * Cut count nubmer of pages from the pool and put them to return list
+ * Cut 'count' number of pages from the pool and put them on the return list.
 *
- * @return count of pages still to allocate to fill the request.
+ * @return count of pages still required to fulfill the request.
 */
 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                struct list_head *pages, int ttm_flags,
@@ -637,7 +637,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                goto out;
        }
        /* find the last pages to include for requested number of pages. Split
-         * pool to begin and halves to reduce search space. */
+         * pool to begin and halve it to reduce search space. */
        if (count <= pool->npages/2) {
                i = 0;
                list_for_each(p, &pool->list) {
@@ -651,7 +651,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
                                break;
                }
        }
-        /* Cut count number of pages from pool */
+        /* Cut 'count' number of pages from the pool */
        list_cut_position(pages, &pool->list, p);
        pool->npages -= count;
        count = 0;
diff --git a/include/acpi/video.h b/include/acpi/video.h
index 0e98e679d3a7..61109f2609fc 100644
--- a/include/acpi/video.h
+++ b/include/acpi/video.h
@@ -5,6 +5,8 @@
 struct acpi_device;
+#define ACPI_VIDEO_CLASS        "video"
 #define ACPI_VIDEO_DISPLAY_CRT  1
 #define ACPI_VIDEO_DISPLAY_TV   2
 #define ACPI_VIDEO_DISPLAY_DVI  3
diff --git a/include/drm/drmP.h b/include/drm/drmP.h
index 738b3a5faa12..d9c8c6c4639a 100644
--- a/include/drm/drmP.h
+++ b/include/drm/drmP.h
@@ -886,6 +886,8 @@ struct drm_driver {
         */
        int (*gem_init_object) (struct drm_gem_object *obj);
        void (*gem_free_object) (struct drm_gem_object *obj);
+        int (*gem_open_object) (struct drm_gem_object *, struct drm_file *);
+        void (*gem_close_object) (struct drm_gem_object *, struct drm_file *);
        /* vga arb irq handler */
        void (*vgaarb_irq)(struct drm_device *dev, bool state);
@@ -1672,13 +1674,6 @@ static __inline__ int drm_pci_device_is_agp(struct drm_device *dev)
        return pci_find_capability(dev->pdev, PCI_CAP_ID_AGP);
 }
-static __inline__ int drm_pci_device_is_pcie(struct drm_device *dev)
-{
-        return pci_find_capability(dev->pdev, PCI_CAP_ID_EXP);
-}
 extern int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver);
 extern void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver);
 extern int drm_get_pci_dev(struct pci_dev *pdev,
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index 62a0e4c4ceee..42e346985186 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -662,9 +662,6 @@ extern int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long start_page,
 extern void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map);
-#if 0
-#endif
 /**
 * ttm_fbdev_mmap - mmap fbdev memory backed by a ttm buffer object.
 *
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 09af2d746d1c..94eb1434316e 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -78,7 +78,7 @@ struct ttm_backend_func {
         *
         * Bind the backend pages into the aperture in the location
         * indicated by @bo_mem. This function should be able to handle
-         * differences between aperture- and system page sizes.
+         * differences between aperture and system page sizes.
         */
        int (*bind) (struct ttm_backend *backend, struct ttm_mem_reg *bo_mem);
@@ -88,7 +88,7 @@ struct ttm_backend_func {
         * @backend: Pointer to a struct ttm_backend.
         *
         * Unbind previously bound backend pages. This function should be
-         * able to handle differences between aperture- and system page sizes.
+         * able to handle differences between aperture and system page sizes.
         */
        int (*unbind) (struct ttm_backend *backend);
@@ -786,7 +786,7 @@ extern int ttm_bo_device_release(struct ttm_bo_device *bdev);
 * ttm_bo_device_init
 *
 * @bdev: A pointer to a struct ttm_bo_device to initialize.
- * @mem_global: A pointer to an initialized struct ttm_mem_global.
+ * @glob: A pointer to an initialized struct ttm_bo_global.
 * @driver: A pointer to a struct ttm_bo_driver set up by the caller.
 * @file_page_offset: Offset into the device address space that is available
 * for buffer data. This ensures compatibility with other users of the
diff --git a/include/drm/ttm/ttm_memory.h b/include/drm/ttm/ttm_memory.h
index b199170b3c2c..26c1f78d136f 100644
--- a/include/drm/ttm/ttm_memory.h
+++ b/include/drm/ttm/ttm_memory.h
@@ -41,7 +41,7 @@
 * @do_shrink: The callback function.
 *
 * Arguments to the do_shrink functions are intended to be passed using
- * inheritance. That is, the argument class derives from struct ttm_mem_srink,
+ * inheritance. That is, the argument class derives from struct ttm_mem_shrink,
 * and can be accessed using container_of().
 */
diff --git a/include/drm/ttm/ttm_object.h b/include/drm/ttm/ttm_object.h
index 0d9db099978b..e46054e5255b 100644
--- a/include/drm/ttm/ttm_object.h
+++ b/include/drm/ttm/ttm_object.h
@@ -111,7 +111,7 @@ struct ttm_object_device;
 *
 * @ref_obj_release: A function to be called when a reference object
 * with another ttm_ref_type than TTM_REF_USAGE is deleted.
- * this function may, for example, release a lock held by a user-space
+ * This function may, for example, release a lock held by a user-space
 * process.
 *
 * This struct is intended to be used as a base struct for objects that
@@ -172,7 +172,7 @@ extern struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file
 /**
 * ttm_base_object_unref
 *
- * @p_base: Pointer to a pointer referncing a struct ttm_base_object.
+ * @p_base: Pointer to a pointer referencing a struct ttm_base_object.
 *
 * Decrements the base object refcount and clears the pointer pointed to by
 * p_base.
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 8062890f725e..129de12353f1 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -32,7 +32,7 @@
 /**
 * Get count number of pages from pool to pages list.
 *
- * @pages: heado of empty linked list where pages are filled.
+ * @pages: head of empty linked list where pages are filled.
 * @flags: ttm flags for page allocation.
 * @cstate: ttm caching state for the page.
 * @count: number of pages to allocate.
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index 11be48e0d168..6216115c7789 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -324,11 +324,16 @@ static inline unsigned int cpufreq_get(unsigned int cpu)
 /* query the last known CPU freq (in kHz). If zero, cpufreq couldn't detect it */
 #ifdef CONFIG_CPU_FREQ
 unsigned int cpufreq_quick_get(unsigned int cpu);
+unsigned int cpufreq_quick_get_max(unsigned int cpu);
 #else
 static inline unsigned int cpufreq_quick_get(unsigned int cpu)
 {
        return 0;
 }
+static inline unsigned int cpufreq_quick_get_max(unsigned int cpu)
+{
+        return 0;
+}
 #endif